[Model] add GLM-5.3-Flash support (#53906)

This commit is contained in:
Jiangyun Zhu
2026-09-03 09:40:35 -07:00
committed by GitHub
parent da8ec28268
commit 98ed0856f3
94 changed files with 17927 additions and 266 deletions
@@ -9,13 +9,23 @@ from vllm.platforms import current_platform
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
if not current_platform.has_device_capability(100):
if not current_platform.is_cuda() or not current_platform.has_device_capability(90):
pytest.skip(
reason="FlashInfer MLA Requires compute capability of 10 or above.",
reason="FlashInfer MLA requires CUDA compute capability 9.0 or above.",
allow_module_level=True,
)
else:
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
from flashinfer.mla import BatchMLAPagedAttentionWrapper
requires_sm90 = pytest.mark.skipif(
not current_platform.is_device_capability_family(90),
reason="This test requires an SM90 GPU.",
)
requires_sm10x = pytest.mark.skipif(
not current_platform.is_device_capability_family(100),
reason="This test requires an SM10x GPU.",
)
# Deepseek R1 MLA config.
NUM_HEADS = 128
@@ -82,6 +92,7 @@ def ref_mla(
@pytest.mark.parametrize("dtype", [torch.bfloat16])
@pytest.mark.parametrize("bs", [1, 2, 4, 16])
@pytest.mark.parametrize("block_size", [32, 64])
@requires_sm10x
def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
torch.set_default_device("cuda")
torch.manual_seed(42)
@@ -119,6 +130,161 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
@requires_sm10x
def test_flashinfer_trtllm_sparse_mla_decode_without_rope():
"""The native sparse MLA path supports a zero-width rotary tail."""
torch.set_default_device("cuda")
torch.manual_seed(42)
batch_size = 2
block_size = 64
num_blocks = 4
sparse_topk = 128
valid_lens = torch.tensor([17, 73], dtype=torch.int32)
query = torch.randn(
batch_size,
1,
NUM_HEADS,
KV_LORA_RANK,
dtype=torch.bfloat16,
)
kv_cache = torch.randn(
num_blocks,
block_size,
KV_LORA_RANK,
dtype=torch.bfloat16,
)
num_slots = num_blocks * block_size
slot_tables = torch.stack(
[torch.randperm(num_slots)[:sparse_topk] for _ in range(batch_size)]
).to(torch.int32)
for row, valid_len in zip(slot_tables, valid_lens.tolist()):
row[valid_len:] = -1
workspace_buffer = torch.empty(
FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.int8,
)
out = trtllm_batch_decode_with_kv_cache_mla(
query=query,
kv_cache=kv_cache.unsqueeze(1),
workspace_buffer=workspace_buffer,
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
kv_lora_rank=KV_LORA_RANK,
qk_rope_head_dim=0,
block_tables=slot_tables.unsqueeze(1),
seq_lens=valid_lens,
max_seq_len=sparse_topk,
sparse_mla_top_k=sparse_topk,
sparse_mla_top_k_lens=valid_lens,
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
bmm2_scale=1.0,
).squeeze(1)
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
refs = []
for batch_idx, valid_len in enumerate(valid_lens.tolist()):
selected_kv = flat_cache[slot_tables[batch_idx, :valid_len].long()]
scores = torch.einsum("hd,kd->hk", query[batch_idx, 0].float(), selected_kv)
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
ref = torch.stack(refs).to(torch.bfloat16)
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
@requires_sm90
def test_flashinfer_sm90_fp8_mla_decode_without_rope():
"""Hopper FA3 supports BF16 queries over an FP8 cache without KPE."""
torch.manual_seed(42)
device = torch.device("cuda")
batch_size = 2
num_heads = 16
page_size = 16
num_pages = 6
q_nope = torch.randn(
batch_size,
num_heads,
KV_LORA_RANK,
dtype=torch.bfloat16,
device=device,
)
q_pe = torch.empty(
batch_size,
num_heads,
0,
dtype=torch.bfloat16,
device=device,
)
ckv = torch.randn(
num_pages,
page_size,
KV_LORA_RANK,
device=device,
)
fp8_max = torch.finfo(torch.float8_e4m3fn).max
ckv_scale = ckv.abs().max().item() / fp8_max
ckv_fp8 = (ckv / ckv_scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
scale_bf16 = torch.tensor(ckv_scale, dtype=torch.bfloat16, device=device)
ckv_ref = ckv_fp8.to(torch.bfloat16) * scale_bf16
kpe_fp8 = torch.empty(
num_pages,
page_size,
0,
dtype=torch.float8_e4m3fn,
device=device,
)
kpe_ref = torch.empty(
num_pages,
page_size,
0,
dtype=torch.bfloat16,
device=device,
)
qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device)
kv_lens = torch.tensor([45, 29], dtype=torch.int32, device=device)
sm_scale = QK_NOPE_HEAD_DIM**-0.5
def run(
ckv_cache: torch.Tensor,
kpe_cache: torch.Tensor,
**kwargs,
) -> torch.Tensor:
workspace = torch.empty(
FLASHINFER_WORKSPACE_BUFFER_SIZE,
dtype=torch.uint8,
device=device,
)
wrapper = BatchMLAPagedAttentionWrapper(workspace, backend="fa3")
wrapper.plan(
qo_indptr,
kv_indptr,
kv_indices,
kv_lens,
num_heads,
KV_LORA_RANK,
0,
page_size,
False,
sm_scale,
q_data_type=torch.bfloat16,
kv_data_type=ckv_cache.dtype,
)
return wrapper.run(q_nope, q_pe, ckv_cache, kpe_cache, **kwargs)
out_ref = run(ckv_ref, kpe_ref)
out = run(ckv_fp8, kpe_fp8, ckv_scale=ckv_scale, kpe_scale=1.0)
torch.testing.assert_close(out, out_ref, atol=2e-2, rtol=2e-2)
@requires_sm10x
def test_flashinfer_mla_decode_workspace_supports_autotune():
"""vLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
@@ -41,6 +41,8 @@ def _make_builder():
builder.device = torch.device("cpu")
builder.kv_cache_spec = SimpleNamespace(block_size=1)
builder.model_dtype = torch.bfloat16
builder.kv_cache_dtype = "fp8"
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
builder.topk_tokens = topk_tokens
builder.req_id_per_token_buffer = torch.zeros(
max_num_batched_tokens, dtype=torch.int32, device="cpu"
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
if current_platform.is_rocm():
pytest.skip(
reason="FlashInfer GDN prefill is not supported on ROCm.",
allow_module_level=True,
)
import flashinfer.gdn_prefill # noqa: E402
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
fi_chunk_gated_delta_rule,
) # noqa: E402
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
captured_cu_seqlens = None
def fake_chunk_gated_delta_rule(**kwargs):
nonlocal captured_cu_seqlens
captured_cu_seqlens = kwargs["cu_seqlens"]
return kwargs["q"]
monkeypatch.setattr(
flashinfer.gdn_prefill,
"chunk_gated_delta_rule",
fake_chunk_gated_delta_rule,
)
q = torch.zeros(1, 2, 1, 2)
cu_seqlens = torch.tensor([0, 2], dtype=torch.int32)
output, final_state = fi_chunk_gated_delta_rule(
q=q,
k=q,
v=q,
g=torch.zeros(1, 2, 1),
beta=torch.zeros(1, 2, 1),
initial_state=torch.zeros(1, 1, 2, 2),
output_final_state=False,
cu_seqlens=cu_seqlens,
use_qk_l2norm_in_kernel=False,
)
assert captured_cu_seqlens is not None
assert captured_cu_seqlens.dtype == torch.int64
assert output.shape == q.shape
assert final_state is None
@@ -0,0 +1,549 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for the batched kpool decode-update kernel.
Validates ``kpool_decode_update_and_maybe_write_cache_batched`` against an
independent pure-torch reference that replicates the per-request, in-position
order semantics: stash each token into a paged tail ring; on pool completion
(``pos % pool_size == pool_size-1``) softmax(gate+ape)-weighted sum + Hadamard-128
+ per-vector fp8 absmax quant + write to the indexer K cache. Covers
no-completion, completion-at-end, completion-mid-batch, non-uniform padding,
plain decode, plus a randomized fuzz pass.
The kernel iterates each request's ``next_n`` tokens in position order inside
one program (grid = num_requests) to preserve the pool-completion
read-after-stash dependency; the reference mirrors that ordering.
``test_decode_writer_matches_prefill_writer`` is deliberately NOT
reference-based: it checks the decode writer against the *prefill* writer
(``kpool_compress_and_write_cache``), the invariant that actually matters in
production. A hand-written reference can drift to match a buggy kernel -- that
is exactly how the stash-gating bug (intra-pool tokens never entering the tail
ring, because the stash was gated on the pool-granular ``slot_mapping``) stayed
green here.
"""
import math
import pytest
import torch
from vllm.platforms import current_platform
if current_platform.is_rocm():
from vllm.models.glm5next.amd.ops.kpool_compress import (
kpool_compress_and_write_cache,
kpool_decode_update_and_maybe_write_cache_batched,
kpool_seed_tail_cache,
)
else:
from vllm.models.glm5next.nvidia.ops.kpool_compress import (
kpool_compress_and_write_cache,
kpool_decode_update_and_maybe_write_cache_batched,
kpool_seed_tail_cache,
)
HEAD_DIM = 128
POOL_SIZE = 16
PAGE_SIZE = 64
NUM_BLOCKS = 32
ROUND_SCALE = True
FP8_DTYPE = current_platform.fp8_dtype()
FP8_MAX = torch.finfo(FP8_DTYPE).max
def _make_caches():
kv = torch.zeros(
NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda"
)
tail = torch.zeros(
NUM_BLOCKS, 2, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
)
return kv, tail
def _tail_slot_for(blocks, pos):
"""tail_slot = block*POOL + pos%POOL; each request owns a distinct tail block."""
blk = torch.tensor(blocks, device=pos.device, dtype=torch.int32).unsqueeze(1)
return (blk * POOL_SIZE + pos % POOL_SIZE).to(torch.int32)
def _seed_prior(tail, blocks, n_prior, seed=42):
if n_prior <= 0:
return
g = torch.Generator(device=tail.device).manual_seed(seed)
prior_k = torch.randn(
len(blocks),
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device=tail.device,
generator=g,
)
prior_s = torch.randn(
len(blocks),
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device=tail.device,
generator=g,
)
for i, blk in enumerate(blocks):
tail[blk, 0, :n_prior, :] = prior_k[i]
tail[blk, 1, :n_prior, :] = prior_s[i]
def _hadamard128_torch(x: torch.Tensor) -> torch.Tensor:
"""Reference Hadamard-128 on the last dim (must be 128)."""
n = x.shape[-1]
assert n == 128
h = torch.tensor([[1.0, 1.0], [1.0, -1.0]], dtype=torch.float32, device=x.device)
while h.shape[0] < n:
h = torch.cat([torch.cat([h, h], dim=1), torch.cat([h, -h], dim=1)], dim=0)
h = h / math.sqrt(n)
return x @ h
def _torch_reference(
kv: torch.Tensor,
tail: torch.Tensor,
tail_slot: torch.Tensor,
key: torch.Tensor,
score: torch.Tensor,
ape: torch.Tensor,
slot_map: torch.Tensor,
pos: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Independent reference for the batched decode-update kernel.
For each request, iterate its next_n tokens in order. On each token:
- if pos%POOL == POOL-1 and pos_valid: compress the pool (slots
[pool_start..pool_start+POOL-1], current token via is_current) and write
fp8 K + fp32 scale to kv_cache at cache_loc.
- always stash the current token's K/score into tail[block, pos%POOL].
"""
kv = kv.clone()
tail = tail.clone()
B, next_n = pos.shape
# The indexer K cache is [num_blocks, PAGE_SIZE, HEAD_DIM+4] uint8 but the
# kernels interpret each page as [HEAD_DIM*PAGE_SIZE bytes of K (token-major)
# | 4*PAGE_SIZE bytes of fp32 scale (token-major)]. Operate on a flat byte
# view so the reference writes K and scale at the exact offsets the kernel
# uses (page_base + tok*HEAD_DIM for K; page_base + HEAD_DIM*PAGE_SIZE +
# tok*4 for scale).
page_bytes = PAGE_SIZE * (HEAD_DIM + 4)
k_region = HEAD_DIM * PAGE_SIZE
tail_slot_cpu = tail_slot.cpu().tolist()
slot_map_cpu = slot_map.cpu().tolist()
pos_cpu = pos.cpu().tolist()
key_cpu = key.float().cpu()
score_cpu = score.float().cpu()
ape_cpu = ape.cpu()
tail_cpu = tail.float().cpu()
kv_flat = kv.view(torch.uint8).reshape(-1).cpu()
for b in range(B):
for t in range(next_n):
cache_loc = slot_map_cpu[b][t]
p = pos_cpu[b][t]
pos_valid = cache_loc >= 0 and p >= 0
safe_pos = max(p, 0)
slot = safe_pos % POOL_SIZE
phys_slot = safe_pos % POOL_SIZE
# Per-token block derivation (a leading invalid sentinel must not
# poison the base for the rest of the request); clamped like the
# kernel so an invalid entry can't form a negative base.
block = max(tail_slot_cpu[b][t], 0) // POOL_SIZE
cur_key = key_cpu[b, t]
cur_score = score_cpu[b, t]
if pos_valid and slot == POOL_SIZE - 1:
pool_logical_start = safe_pos - slot
pool_scores = []
pool_ks = []
for ps in range(POOL_SIZE):
is_current = ps == slot
phys = (pool_logical_start + ps) % POOL_SIZE
if is_current:
s = cur_score
k = cur_key
else:
s = tail_cpu[block, 1, phys]
k = tail_cpu[block, 0, phys]
s = s + ape_cpu[ps]
pool_scores.append(s)
pool_ks.append(k)
pool_scores = torch.stack(pool_scores) # [POOL, D]
pool_ks = torch.stack(pool_ks) # [POOL, D]
max_score = pool_scores.max(dim=0).values
prob = torch.exp(pool_scores - max_score)
denom = prob.sum(dim=0)
acc = (pool_ks * prob).sum(dim=0)
x = (acc / denom).to(torch.bfloat16).to(torch.float32)
x = _hadamard128_torch(x).to(torch.bfloat16).to(torch.float32)
absmax = torch.clamp(x.abs().max(), min=1e-4)
if ROUND_SCALE:
scale = torch.exp2(torch.ceil(torch.log2(absmax / FP8_MAX)))
else:
scale = absmax / FP8_MAX
quantized = torch.clamp(x / scale, -FP8_MAX, FP8_MAX).to(FP8_DTYPE)
# write K and scale at the separated-layout offsets
loc = cache_loc
loc_page_index = loc // PAGE_SIZE
loc_tok = loc % PAGE_SIZE
page_base = loc_page_index * page_bytes
if current_platform.is_rocm():
dims = torch.arange(HEAD_DIM)
k_off = (
page_base
+ (loc_tok // 16) * 16 * HEAD_DIM
+ (dims // 16) * 16 * 16
+ (loc_tok % 16) * 16
+ dims % 16
)
else:
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
s_off = page_base + k_region + loc_tok * 4
kv_flat[k_off] = quantized.view(torch.uint8)
kv_flat[s_off : s_off + 4] = scale.detach().reshape(1).view(torch.uint8)
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
# pos_valid keys off the pool-granular cache_loc, which is -1 for
# every token that is not the pool's last, so gating the stash on it
# would drop all intra-pool tokens.
if p >= 0 and tail_slot_cpu[b][t] >= 0:
tail_cpu[block, 0, phys_slot] = cur_key
tail_cpu[block, 1, phys_slot] = cur_score
kv_out = kv_flat.view(NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4).to(device="cuda")
return kv_out, tail_cpu.to(torch.bfloat16).to(device="cuda")
def _assert_eq(r_ref, r_kern):
kv_ref, tail_ref = r_ref
kv_kern, tail_kern = r_kern
assert torch.equal(kv_ref, kv_kern), (
"kv_cache differs: max diff "
f"{(kv_ref.int() - kv_kern.int()).abs().max().item()}"
)
assert torch.equal(tail_ref, tail_kern), (
"tail_kv_cache differs: max diff "
f"{(tail_ref.float() - tail_kern.float()).abs().max().item()}"
)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_amd_prefill_writer_uses_preshuffled_cache_layout():
from vllm.models.glm5next.amd.ops.kpool_compress import (
kpool_compress_and_write_cache as amd_kpool_compress,
)
torch.manual_seed(0)
token_offset = 17
kv = torch.zeros(1, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda")
key = torch.randn(1, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
score = torch.randn_like(key)
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
compressed_k, compressed_scale = amd_kpool_compress(
kv,
key,
score,
ape,
torch.tensor([token_offset], dtype=torch.int64, device="cuda"),
pool_size=POOL_SIZE,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
return_compressed=True,
)
dim = torch.arange(HEAD_DIM, device="cuda")
offsets = (
(token_offset // 16) * 16 * HEAD_DIM
+ (dim // 16) * 16 * 16
+ (token_offset % 16) * 16
+ dim % 16
)
flat = kv.view(torch.uint8).reshape(-1)
stored_k = flat[offsets].view(compressed_k.dtype)
scale_offset = PAGE_SIZE * HEAD_DIM + token_offset * 4
stored_scale = flat[scale_offset : scale_offset + 4].view(torch.float32)
assert torch.equal(stored_k, compressed_k[0])
assert torch.equal(stored_scale, compressed_scale)
def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
kv = kv.clone()
tail = tail.clone()
kpool_decode_update_and_maybe_write_cache_batched(
kv,
tail,
tail_slot,
key,
score,
ape,
slot_map,
pos,
POOL_SIZE,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
return kv, tail
@pytest.mark.parametrize("pool_size", [4, 16])
def test_decode_writer_matches_prefill_writer(pool_size):
"""Compare production decode and prefill writers for pool sizes 4 and 16."""
n_pools, page, nblk = 8, 64, 4
n_tok = n_pools * pool_size
dev = "cuda"
torch.manual_seed(0)
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
ape = torch.randn(pool_size, HEAD_DIM, dtype=torch.float32, device=dev)
kv_prefill = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
kpool_compress_and_write_cache(
kv_prefill,
k.view(n_pools, pool_size, HEAD_DIM),
score.view(n_pools, pool_size, HEAD_DIM),
ape,
torch.arange(n_pools, dtype=torch.int64, device=dev),
pool_size=pool_size,
head_dim=HEAD_DIM,
round_scale=ROUND_SCALE,
)
# One request owning tail block 0, fed one token per decode step.
kv_decode = torch.zeros_like(kv_prefill)
tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev)
for t in range(n_tok):
completes = t % pool_size == pool_size - 1
kpool_decode_update_and_maybe_write_cache_batched(
kv_decode,
tail,
# token-granular: every token has a valid tail slot
torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev),
k[t].view(1, 1, HEAD_DIM),
score[t].view(1, 1, HEAD_DIM),
ape,
# pool-granular: only the pool's last token carries a cache slot
torch.tensor(
[[t // pool_size if completes else -1]], dtype=torch.int32, device=dev
),
torch.tensor([[t]], dtype=torch.int32, device=dev),
pool_size,
HEAD_DIM,
round_scale=ROUND_SCALE,
)
differing = [
p
for p in range(n_pools)
if not torch.equal(
kv_prefill[p // page, p % page], kv_decode[p // page, p % page]
)
]
assert not differing, (
f"decode-written pools differ from prefill-written pools: "
f"{len(differing)}/{n_pools} (pool_size={pool_size}, first={differing[:5]})"
)
def test_leading_invalid_tail_slot():
"""A request whose FIRST token carries an invalid (-1) tail slot while a
later token is a real pool completion.
The tail block must be derived per token, not from token 0: a leading
invalid sentinel would otherwise poison the base address for the whole
request (out-of-bounds tail reads on the completion).
"""
torch.manual_seed(0)
B, next_n, blocks = 2, 4, [3, 5]
# req 0: token 0 invalid (pos -1), tokens 1..3 valid, completion at pos 15
# req 1: all valid, no completion
pos = torch.tensor(
[[-1, 13, 14, 15], [4, 5, 6, 7]], dtype=torch.int32, device="cuda"
)
safe_pos = torch.where(pos >= 0, pos, 0)
tail_slot = _tail_slot_for(blocks, safe_pos)
# leading invalid entry carries the -1 sentinel, as the scatter path emits
tail_slot[0, 0] = -1
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[0, 3] = 15 # req 0 completes its pool on the last verify token
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
kv, tail = _make_caches()
_seed_prior(tail, blocks, 13)
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
_assert_eq(r_ref, r_kern)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_amd_prefill_seed_honors_padded_tail_block_stride():
"""The tail shares a padded indexer allocation in production."""
kpool = 4
num_blocks = 6
logical_block_elems = 2 * kpool * HEAD_DIM
padded_block_elems = logical_block_elems + 256
sentinel = -123.0
backing = torch.full(
(num_blocks * padded_block_elems,),
sentinel,
dtype=torch.bfloat16,
device="cuda",
)
tail = torch.as_strided(
backing,
size=(num_blocks, 2, kpool, HEAD_DIM),
stride=(padded_block_elems, kpool * HEAD_DIM, HEAD_DIM, 1),
)
block = 3
ring_slot = 2
key = torch.arange(HEAD_DIM, dtype=torch.bfloat16, device="cuda").unsqueeze(0)
score = (key + 256).to(torch.bfloat16)
tail_slot = torch.tensor(
[block * kpool + ring_slot], dtype=torch.int32, device="cuda"
)
kpool_seed_tail_cache(tail, key, score, tail_slot, kpool, HEAD_DIM)
torch.accelerator.synchronize()
assert torch.equal(tail[block, 0, ring_slot], key[0])
assert torch.equal(tail[block, 1, ring_slot], score[0])
compact_offset = (block * 2 * kpool + ring_slot) * HEAD_DIM
assert torch.all(backing[compact_offset : compact_offset + HEAD_DIM] == sentinel)
@pytest.mark.parametrize(
"case_id",
[
"no_completion",
"completion_at_end",
"completion_mid_batch",
"non_uniform_padding",
"plain_decode",
],
)
def test_batched_matches_reference(case_id):
torch.manual_seed(0)
if case_id == "no_completion":
B, next_n, blocks = 3, 4, [0, 1, 2]
pos = (
torch.arange(next_n, device="cuda", dtype=torch.int32)
.unsqueeze(0)
.expand(B, -1)
.contiguous()
)
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
n_prior = 0
elif case_id == "completion_at_end":
B, next_n, blocks = 2, 4, [0, 1]
pos = torch.tensor(
[[12, 13, 14, 15], [12, 13, 14, 15]], dtype=torch.int32, device="cuda"
)
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[:, 3] = torch.tensor(
[15, PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
)
n_prior = POOL_SIZE - next_n
elif case_id == "completion_mid_batch":
B, next_n, blocks = 3, 4, [0, 1, 2]
pos = torch.tensor([[13, 14, 15, 16]] * B, dtype=torch.int32, device="cuda")
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[:, 2] = torch.tensor(
[15, PAGE_SIZE + 15, 2 * PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
)
n_prior = 13
elif case_id == "non_uniform_padding":
B, next_n, blocks = 2, 4, [0, 1]
pos = torch.tensor(
[[12, 13, 14, 15], [12, 13, -1, -1]], dtype=torch.int32, device="cuda"
)
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
slot_map[0, 3] = 15
n_prior = POOL_SIZE - 4
else: # plain_decode
B, next_n, blocks = 4, 1, [0, 1, 2, 3]
pos = torch.tensor([[5], [6], [7], [8]], dtype=torch.int32, device="cuda")
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
n_prior = 0
if case_id == "non_uniform_padding":
safe_pos = torch.where(pos >= 0, pos, 0)
tail_slot = torch.where(pos >= 0, _tail_slot_for(blocks, safe_pos), 0)
else:
tail_slot = _tail_slot_for(blocks, pos)
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
kv, tail = _make_caches()
_seed_prior(tail, blocks, n_prior)
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
_assert_eq(r_ref, r_kern)
@pytest.mark.parametrize("seed", list(range(20)))
def test_batched_matches_reference_fuzz(seed):
"""Random B / next_n / start positions; covers 0, 1, and multi completion."""
g = torch.Generator(device="cuda").manual_seed(seed)
B = int(torch.randint(1, 6, (1,), generator=g, device="cuda").item())
next_n = int(torch.randint(1, 8, (1,), generator=g, device="cuda").item())
blocks = list(range(B))
starts = torch.randint(0, 33, (B,), generator=g, device="cuda", dtype=torch.int32)
pos = starts.unsqueeze(1) + torch.arange(
next_n, device="cuda", dtype=torch.int32
).unsqueeze(0)
tail_slot = _tail_slot_for(blocks, pos)
is_completion = pos % POOL_SIZE == POOL_SIZE - 1
blk = torch.tensor(blocks, device="cuda", dtype=torch.int32).unsqueeze(1)
pool_slot = blk * PAGE_SIZE + (POOL_SIZE - 1)
slot_map = torch.where(is_completion, pool_slot, torch.full_like(pos, -1))
key = torch.randn(
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
)
score = torch.randn(
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
)
ape = torch.randn(
POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda", generator=g
)
kv, tail = _make_caches()
prior_g = torch.Generator(device="cuda").manual_seed(seed + 1000)
for b in range(B):
n_prior = int(starts[b].item()) % POOL_SIZE
if n_prior > 0:
pk = torch.randn(
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
generator=prior_g,
)
ps = torch.randn(
n_prior,
HEAD_DIM,
dtype=torch.bfloat16,
device="cuda",
generator=prior_g,
)
tail[blocks[b], 0, :n_prior, :] = pk
tail[blocks[b], 1, :n_prior, :] = ps
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
_assert_eq(r_ref, r_kern)
+225 -1
View File
@@ -5,13 +5,23 @@ from types import SimpleNamespace
import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F
import vllm.model_executor.kernels.mhc # noqa: F401
import vllm.model_executor.layers.mhc as mhc_layers
from vllm.model_executor.kernels.mhc.tilelang import (
_tilelang_hc_prenorm_gemm,
_torch_hc_prenorm_gemm,
)
from vllm.model_executor.layers.mhc import HAS_TILELANG_MHC
from vllm.model_executor.layers.mhc import (
HAS_AITER_MHC,
HAS_AITER_MHC_FUSED,
HAS_AITER_MHC_FUSED_NORM,
HAS_AITER_MHC_PRE_NORM,
HAS_TILELANG_MHC,
MHCFusedPostPreOp,
MHCPreOp,
)
from vllm.models.deepseek_v4.nvidia.model import (
DeepseekV4DecoderLayer,
DeepseekV4Model,
@@ -291,6 +301,220 @@ def test_mhc_fused_post_pre(num_tokens, hidden_size, hc_mult):
torch.testing.assert_close(x, layer_input_ref, atol=1e-2, rtol=1e-2)
def _rocm_mhc_inputs(num_tokens=2, hidden_size=256, hc_mult=4):
residual = torch.randn(
(num_tokens, hc_mult, hidden_size), dtype=torch.bfloat16, device=DEVICE
)
hc_mult3 = 2 * hc_mult + hc_mult * hc_mult
fn = (
torch.randn(
(hc_mult3, hc_mult * hidden_size), dtype=torch.float32, device=DEVICE
)
* 1e-4
)
hc_scale = torch.randn((3,), dtype=torch.float32, device=DEVICE) * 0.1
hc_base = torch.randn((hc_mult3,), dtype=torch.float32, device=DEVICE) * 0.1
norm_weight = torch.randn(hidden_size, dtype=torch.bfloat16, device=DEVICE)
return residual, fn, hc_scale, hc_base, norm_weight
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_mhc_pre_rocm_fallback_applies_norm(monkeypatch):
set_random_seed(0)
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs()
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
sinkhorn_repeat = 20
hc_post_alpha = 1.0
ref = mhc_pre_ref(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
)
expected_layer_input = F.rms_norm(
ref[2], (ref[2].shape[-1],), norm_weight, norm_eps
)
monkeypatch.setattr(mhc_layers, "HAS_AITER_MHC", True)
monkeypatch.setattr(mhc_layers, "HAS_AITER_MHC_PRE_NORM", False)
monkeypatch.setattr(mhc_layers, "HAS_TILELANG_MHC", False)
out = object.__new__(MHCPreOp).forward_hip(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
torch.testing.assert_close(out[0], ref[0])
torch.testing.assert_close(out[1], ref[1])
torch.testing.assert_close(out[2], expected_layer_input)
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_mhc_fused_rocm_fallback_applies_norm(monkeypatch):
set_random_seed(0)
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs()
x = torch.randn((2, 256), dtype=torch.bfloat16, device=DEVICE)
post_layer_mix = torch.randn((2, 4, 1), dtype=torch.float32, device=DEVICE)
comb_res_mix = torch.randn((2, 4, 4), dtype=torch.float32, device=DEVICE)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
sinkhorn_repeat = 20
hc_post_alpha = 1.0
residual_ref = mhc_post_ref(x, residual, post_layer_mix, comb_res_mix)
pre_ref = mhc_pre_ref(
residual_ref,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
)
expected_layer_input = F.rms_norm(
pre_ref[2], (pre_ref[2].shape[-1],), norm_weight, norm_eps
)
monkeypatch.setattr(mhc_layers, "HAS_AITER_MHC_FUSED", False)
monkeypatch.setattr(mhc_layers, "HAS_TILELANG_MHC", False)
out = object.__new__(MHCFusedPostPreOp).forward_hip(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
torch.testing.assert_close(out[0], residual_ref)
torch.testing.assert_close(out[1], pre_ref[0])
torch.testing.assert_close(out[2], pre_ref[1])
torch.testing.assert_close(out[3], expected_layer_input)
@pytest.mark.skipif(
not (current_platform.is_rocm() and HAS_AITER_MHC and HAS_AITER_MHC_PRE_NORM),
reason="AITER mHC with fused RMSNorm required",
)
def test_mhc_pre_rocm_aiter_fuses_norm():
set_random_seed(0)
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
num_tokens=2, hidden_size=7168
)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
sinkhorn_repeat = 20
hc_post_alpha = 1.0
ref = mhc_pre_ref(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
)
expected_layer_input = F.rms_norm(
ref[2], (ref[2].shape[-1],), norm_weight, norm_eps
)
out = object.__new__(MHCPreOp).forward_hip(
residual,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
torch.testing.assert_close(out[0], ref[0], atol=5e-2, rtol=1e-2)
torch.testing.assert_close(out[1], ref[1], atol=5e-2, rtol=1e-2)
torch.testing.assert_close(out[2], expected_layer_input, atol=5e-2, rtol=1e-2)
@pytest.mark.skipif(
not (
current_platform.is_rocm() and HAS_AITER_MHC_FUSED and HAS_AITER_MHC_FUSED_NORM
),
reason="AITER fused mHC with RMSNorm required",
)
def test_mhc_fused_rocm_aiter_fuses_norm():
set_random_seed(0)
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
num_tokens=2, hidden_size=7168
)
x = torch.randn((2, 7168), dtype=torch.bfloat16, device=DEVICE)
post_layer_mix = torch.randn((2, 4, 1), dtype=torch.float32, device=DEVICE)
comb_res_mix = torch.randn((2, 4, 4), dtype=torch.float32, device=DEVICE)
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
sinkhorn_repeat = 20
hc_post_alpha = 1.0
residual_ref = mhc_post_ref(x, residual, post_layer_mix, comb_res_mix)
pre_ref = mhc_pre_ref(
residual_ref,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
)
expected_layer_input = F.rms_norm(
pre_ref[2], (pre_ref[2].shape[-1],), norm_weight, norm_eps
)
out = object.__new__(MHCFusedPostPreOp).forward_hip(
x,
residual,
post_layer_mix,
comb_res_mix,
fn,
hc_scale,
hc_base,
rms_eps,
hc_pre_eps,
hc_sinkhorn_eps,
hc_post_alpha,
sinkhorn_repeat,
norm_weight=norm_weight,
norm_eps=norm_eps,
)
torch.testing.assert_close(out[0], residual_ref, atol=5e-2, rtol=1e-2)
torch.testing.assert_close(out[1], pre_ref[0], atol=5e-2, rtol=1e-2)
torch.testing.assert_close(out[2], pre_ref[1], atol=5e-2, rtol=1e-2)
torch.testing.assert_close(out[3], expected_layer_input, atol=5e-2, rtol=1e-2)
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm required",
+6
View File
@@ -294,6 +294,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
),
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
"GPTBigCodeForCausalLM": _HfExamplesInfo(
"bigcode/starcoder",
@@ -933,6 +934,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
"H2OVLChatModel": _HfExamplesInfo(
"h2oai/h2ovl-mississippi-800m",
trust_remote_code=True,
@@ -1703,6 +1705,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
speculative_model="zai-org/GLM-OCR",
min_transformers_version="5.1.0",
),
"Glm5NextMTPModel": _HfExamplesInfo(
"zai-org/GLM-5.3-Flash",
speculative_model="zai-org/GLM-5.3-Flash",
),
"HYV3MTPModel": _HfExamplesInfo(
"tencent/Hy3-preview",
speculative_model="tencent/Hy3-preview",
+1 -1
View File
@@ -142,7 +142,7 @@ def can_initialize(
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
monkeypatch.context() as m,
):
if requires_spawn_multiprocessing():
if requires_spawn_multiprocessing() or model_arch == "Glm5NextForCausalLM":
# The EngineCore subprocess re-imports the class and does not
# inherit the KV-cache patch above, so it OOMs. Run in-process
# so the patch applies.
+231
View File
@@ -20,6 +20,7 @@ from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
DynamicVideoBackend,
Glm5NextVideoBackend,
GLM46VVideoBackend,
Molmo2VideoBackend,
Qwen2VLVideoBackend,
@@ -1470,3 +1471,233 @@ def test_glm46v_duration_estimation_from_fps():
assert len(indices) > 0
assert len(indices) % 2 == 0
assert all(0 <= idx < 90 for idx in indices)
def test_glm5next_backend_selected_for_processor():
"""Glm5NextVideoProcessor maps to the glm5next loader so only the
sampled frames are decoded instead of the whole container. Both the
borrowed-config spelling and the dedicated Glm5next class name (landing
with the new checkpoint) must resolve."""
for name in ("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"):
assert VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(name) == "glm5next"
@pytest.mark.parametrize(
("total_frames", "original_fps", "duration", "fps", "max_frames"),
[
(900, 30.0, 30.0, -1, None), # 30s at flat 2.0 raw fps -> 60 frames
(3000, 30.0, 100.0, -1, None),
(72000, 30.0, 2400.0, -1, None), # 2048 cap
(48, 2.0, 24.0, -1, None),
(7, 30.0, 10.0, -1, None), # short video -> uniform spread + dedup
(300, 25.0, 0, -1, None), # duration derived from frame count
(900, 30.0, 30.0, 4, None), # request fps override (raw fps)
(900, 30.0, 30.0, -1, 16), # request max_frames override
],
)
def test_glm5next_backend_indices_match_sampler(
total_frames, original_fps, duration, fps, max_frames
):
"""The loader must select exactly the frames the processor's sampler
would, with target.fps mapping onto the raw-fps override."""
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
source = VideoSourceMetadata(
total_frames_num=total_frames, original_fps=original_fps, duration=duration
)
target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=-1)
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(
source, target, max_frames=max_frames
)
assert indices == glm_sample_frame_indices(
total_frames,
original_fps,
duration,
target_fps=fps if fps > 0 else None,
max_frame_count=max_frames,
)
assert len(indices) % 2 == 0
assert indices == sorted(indices) # pair padding may repeat the last frame
assert all(0 <= idx < total_frames for idx in indices)
def test_glm5next_backend_metadata_contract():
"""create_hf_metadata reports the subset so the processor skips
re-sampling (do_sample_frames=False) and keeps the original totals."""
source = VideoSourceMetadata(total_frames_num=900, original_fps=30.0, duration=30.0)
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(source, target)
metadata = Glm5NextVideoBackend.create_hf_metadata(
source, indices, video_backend="glm5next"
)
assert metadata["do_sample_frames"] is False
assert metadata["frames_indices"] == indices
assert metadata["total_num_frames"] == 900
assert metadata["fps"] == 30.0
assert metadata["duration"] == 30.0
# A fully-selected source keeps do_sample_frames=True so the processor's
# sampler takes over on the complete frame set.
full = list(range(48))
assert (
Glm5NextVideoBackend.create_hf_metadata(
VideoSourceMetadata(total_frames_num=48, original_fps=2.0, duration=24.0),
full,
video_backend="glm5next",
)["do_sample_frames"]
is True
)
def _write_gray_video(tmp_path, total_frames, fps, size=(32, 32)):
"""Synthetic clip whose frame i is flat gray level i (near-lossless under
mp4v), so a decoded frame's level maps back to its source index."""
cv2 = pytest.importorskip("cv2")
path = tmp_path / f"gray_{total_frames}_{fps}.mp4"
writer = cv2.VideoWriter(
str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, size, isColor=False
)
assert writer.isOpened()
for i in range(total_frames):
writer.write(np.full((*size, 1), i, dtype=np.uint8))
writer.release()
return path
class _CountingCap:
"""Proxy over a real capture that counts grab/seek decoding work."""
def __init__(self, cap):
self._cap = cap
self.grabs = 0
self.seeks = 0
def get(self, prop):
return self._cap.get(prop)
def grab(self):
self.grabs += 1
return self._cap.grab()
def set(self, prop, value):
self.seeks += 1
return self._cap.set(prop, value)
def read(self):
return self._cap.read()
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
def test_glm5next_backend_codec_parity(tmp_path, backend):
"""Every codec samples the same GLM indices and decodes the same
frames; the OpenCV seek reader and torchcodec batched index-exact decode
must agree."""
if backend == "torchcodec":
pytest.importorskip("torchcodec")
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
total_frames, fps = 120, 10
path = _write_gray_video(tmp_path, total_frames, fps)
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
for max_frames in (None, 6):
kwargs = {} if max_frames is None else {"max_frames": max_frames}
expected = glm_sample_frame_indices(
total_frames, float(fps), 12.0, max_frame_count=max_frames
)
frames, metadata = Glm5NextVideoBackend.load_bytes(
path.read_bytes(), backend=backend, **kwargs
)
assert metadata["frames_indices"] == expected
assert metadata["video_backend"].startswith(backend)
assert len(frames) == len(expected)
for i, idx in enumerate(expected):
assert abs(round(float(np.asarray(frames[i]).mean())) - idx) <= 1
def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
"""End to end over a synthetic clip: load_bytes returns exactly the
sampler's frame count, with the right frame content at each index."""
pytest.importorskip("cv2")
total_frames, fps = 60, 10
path = _write_gray_video(tmp_path, total_frames, fps)
from vllm.transformers_utils.processors.glm5next import (
glm_sample_frame_indices,
)
expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
assert len(frames) == len(expected)
assert metadata["frames_indices"] == expected
assert metadata["do_sample_frames"] is (len(expected) == total_frames)
# Flat gray frames survive mp4v near-losslessly: each decoded frame's
# level maps back to its source frame index.
for frame, idx in zip(frames, expected):
decoded_idx = round(float(np.asarray(frame).mean()))
assert abs(decoded_idx - idx) <= 1
def test_glm5next_read_frames_seeks_past_large_gaps(tmp_path):
"""Sparse targets must not walk the container: the stock reader grabs
every frame up to the last index; the GLM reader seeks instead."""
cv2 = pytest.importorskip("cv2")
total_frames, fps = 200, 10
path = _write_gray_video(tmp_path, total_frames, fps)
targets = [0, 80, 160, 190] # gaps of 80/80/30 -> only 30 <= threshold
stock = cv2.VideoCapture(str(path))
_, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
stock.release()
assert stock_indices == targets
cap = _CountingCap(cv2.VideoCapture(str(path)))
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
cap._cap.release()
assert indices == targets == stock_indices
for frame, idx in zip(frames, targets):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# Walks only the sub-threshold 30-frame hop; the two 80-frame gaps are
# seeks. The stock reader grabs all 190 preceding frames.
assert cap.grabs <= 29
assert cap.seeks == 3
def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
"""Dense targets keep the sequential walk (seeking would be slower) and
return the same frames as the stock reader."""
cv2 = pytest.importorskip("cv2")
total_frames, fps = 120, 10
path = _write_gray_video(tmp_path, total_frames, fps)
targets = list(range(0, total_frames, 15)) # gaps of 15 -> all walking
stock = cv2.VideoCapture(str(path))
stock_frames, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
stock.release()
cap = _CountingCap(cv2.VideoCapture(str(path)))
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
cap._cap.release()
assert indices == stock_indices
for frame, stock_frame, idx in zip(frames, stock_frames, targets):
assert abs(float(np.asarray(frame).mean()) - float(stock_frame.mean())) <= 2.0
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# One initial seek, then pure walking -- no re-seek churn.
assert cap.seeks == 1
@@ -0,0 +1,389 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor."""
import math
import pytest
import torch
from PIL import Image
from transformers.image_utils import PILImageResampling
from transformers.video_utils import VideoMetadata
from vllm.transformers_utils.processors.glm5next import (
Glm5NextImageProcessor,
Glm5NextVideoProcessor,
_get_pad_content_size,
_resize_or_pad,
glm_sample_frame_indices,
smart_resize,
)
PATCH_SIZE = 14
MERGE_SIZE = 2
PATCH_EXPAND_FACTOR = 1 # checkpoint processor_config.json
FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR # 28
PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2 # 1568
MIN_TOKENS = 16
MAX_TOKENS = 800 # serving-cap equivalent budget, keeps test canvases small
MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN
MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN
def resize(height: int, width: int, t: int = 2) -> tuple[int, int]:
return smart_resize(
t,
height,
width,
t_factor=2,
h_factor=FACTOR,
w_factor=FACTOR,
min_pixels=MIN_PIXELS,
max_pixels=MAX_PIXELS,
)
@pytest.mark.parametrize(
("height", "width", "t", "expected"),
[
(300, 400, 2, (308, 420)), # ceil alignment, no budget pressure
(50, 80, 2, (112, 168)), # below min budget: proportional upscale
(57, 3, 2, (504, 28)), # slim side ceils to one factor
(2160, 3840, 2, (588, 1064)), # shrink: max canvas under the cap
(112, 11200, 2, (84, 7420)), # extreme aspect stays proportional
(300, 400, 16, (252, 308)), # frame count eats into the budget
(112, 11200, 16, (28, 2800)), # slim side stays positive, budget holds
],
)
def test_smart_resize_reference_values(height, width, t, expected):
got = smart_resize(
t,
height,
width,
t_factor=2,
h_factor=FACTOR,
w_factor=FACTOR,
min_pixels=MIN_PIXELS,
max_pixels=MAX_PIXELS,
)
assert got == expected
# The aligned canvas never exceeds the budget it was fitted against.
t_bar = max(2, round(t / 2) * 2)
assert t_bar * got[0] * got[1] <= MAX_PIXELS
def test_smart_resize_stays_snapped_and_positive():
heights = (1, 27, 57, 111, 113, 300, 720, 2160)
widths = (1, 27, 57, 111, 113, 400, 1280, 11200)
for h in heights:
for w in widths:
for t in (2, 4, 16, 64):
resized_h, resized_w = resize(h, w, t)
assert resized_h > 0 and resized_w > 0
assert resized_h % FACTOR == 0
assert resized_w % FACTOR == 0
t_bar = max(2, round(t / 2) * 2)
assert t_bar * resized_h * resized_w <= MAX_PIXELS
@pytest.mark.parametrize(
("height", "width", "match"),
[
(0, 100, "must be positive"),
(100, 0, "must be positive"),
],
)
def test_smart_resize_rejects_degenerate_inputs(height, width, match):
with pytest.raises(ValueError, match=match):
resize(height, width, 2)
def test_smart_resize_rejects_inverted_budget():
with pytest.raises(ValueError, match="min_pixels must be less than or equal"):
smart_resize(
2,
100,
100,
t_factor=2,
h_factor=FACTOR,
w_factor=FACTOR,
min_pixels=MAX_PIXELS,
max_pixels=MIN_PIXELS,
)
def test_get_pad_content_size():
# Oversized content shrinks proportionally, never upscales by default.
assert _get_pad_content_size(300, 400, 308, 420) == (300, 400)
assert _get_pad_content_size(600, 800, 308, 420) == (308, 410)
# allow_upscale enlarges small content toward the canvas.
assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112)
def test_resize_or_pad_pads_right_and_bottom():
stacked = torch.rand(1, 3, 300, 400)
def identity_resize(x, size, resample=None):
assert (size.height, size.width) == (300, 400)
return x
padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize)
assert padded.shape == (1, 3, 308, 420)
torch.testing.assert_close(padded[..., :300, :400], stacked)
assert padded[..., 300:, :].eq(0).all()
assert padded[..., :, 400:].eq(0).all()
def force_resize(x, size, resample=None):
return torch.zeros(x.shape[0], x.shape[1], size.height, size.width)
assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == (
1,
3,
308,
420,
)
with pytest.raises(ValueError, match="resize_mode"):
_resize_or_pad(stacked, 308, 420, "crop", None, identity_resize)
@pytest.fixture(scope="module")
def image_processor():
return Glm5NextImageProcessor(
min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
)
@pytest.mark.parametrize(
("height", "width"),
[(50, 80), (300, 400), (2160, 3840), (112, 11200)],
)
def test_image_grid_matches_smart_resize(image_processor, height, width):
out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
resized_h, resized_w = resize(height, width)
grid = out["image_grid_thw"][0].tolist()
assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE]
assert out["pixel_values"].shape == (
grid[1] * grid[2],
3 * 2 * PATCH_SIZE * PATCH_SIZE,
)
@pytest.mark.parametrize(
("height", "width"),
[(50, 80), (300, 400), (2160, 3840), (112, 11200)],
)
def test_image_patch_count_matches_preprocess(image_processor, height, width):
out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
grid = out["image_grid_thw"][0].tolist()
expected = grid[1] * grid[2]
assert image_processor.get_number_of_image_patches(height, width) == expected
@pytest.fixture(scope="module")
def video_processor():
return Glm5NextVideoProcessor(
min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
)
def run_video_preprocess(video_processor, frames, **overrides):
kwargs = dict(
resample=PILImageResampling.BICUBIC,
image_mean=(0.48145466, 0.4578275, 0.40821073),
image_std=(0.26862954, 0.26130258, 0.27577711),
patch_size=PATCH_SIZE,
temporal_patch_size=2,
merge_size=MERGE_SIZE,
patch_expand_factor=PATCH_EXPAND_FACTOR,
return_tensors="pt",
)
kwargs.update(overrides)
return video_processor._preprocess([frames], **kwargs)
def test_video_preprocess_pads_odd_frame_count(video_processor):
out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400))
grid = out["video_grid_thw"][0].tolist()
# 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420).
assert grid == [4, 22, 30]
assert out["pixel_values_videos"].shape == (
4 * 22 * 30,
3 * 2 * PATCH_SIZE * PATCH_SIZE,
)
def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor):
out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200))
# Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap.
assert out["video_grid_thw"][0].tolist() == [8, 2, 200] # height 28, not 0
def test_video_preprocess_pads_content_not_distort(video_processor):
frames = torch.zeros(4, 3, 300, 400)
frames[..., 50, 60] = 1.0 # marker inside the content area
# Bypass rescale/normalize so zero padding stays exactly zero.
out = run_video_preprocess(
video_processor, frames, do_rescale=False, do_normalize=False
)
# Pad mode keeps the 300x400 content aspect on the (308, 420) canvas
# with zero padding on the right/bottom.
grid = out["video_grid_thw"][0].tolist()
assert grid == [2, 22, 30]
patches = out["pixel_values_videos"].view(
grid[0],
grid[1] // MERGE_SIZE,
grid[2] // MERGE_SIZE,
MERGE_SIZE,
MERGE_SIZE,
3 * 2 * PATCH_SIZE * PATCH_SIZE,
)
frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1)
# Patch columns fully right of the 400px content are pure padding.
assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0
@pytest.mark.parametrize(
("total_frames", "fps", "duration", "expected_len", "first", "last"),
[
# Dense source: the tp-scaled greedy overshoots extract_t, so the
# fixup spreads picks uniformly across the whole video (linspace).
(900, 30.0, 30.0, 60, 0, 899),
(3000, 30.0, 100.0, 200, 0, 2999),
# extract_t capped at 2048 frames.
(72000, 30.0, 2400.0, 2048, 0, 71999),
# Low container fps: the greedy picks every frame.
(48, 2.0, 24.0, 48, 0, 47),
# Duration derived from the frame count when metadata lacks it.
(300, 25.0, 0, 26, 0, 299),
],
)
def test_glm_sample_frame_indices_behaviour(
total_frames, fps, duration, expected_len, first, last
):
indices = glm_sample_frame_indices(total_frames, fps, duration)
assert len(indices) == expected_len
assert indices[0] == first
assert indices[-1] == last
assert len(indices) % 2 == 0
assert indices == sorted(indices)
assert all(0 <= idx < total_frames for idx in indices)
def test_glm_sample_frame_indices_short_clip_floor_spread():
# extract_t (20) > total (7): evenly spaced timestamps, deduplicated,
# then pair-padded to an even count.
assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6]
@pytest.mark.parametrize(
("kwargs", "expected_len"),
[
({"target_fps": 0.5}, 16),
({"max_frame_count": 16}, 16),
({"target_fps": 8}, 240), # 30s * 8 = 240, under the 2048 cap
],
)
def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len):
indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs)
assert len(indices) == expected_len
assert len(indices) % 2 == 0
def test_sample_frames_fps_interval(video_processor):
indices = video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
)
assert len(indices) == 60
assert indices[0] == 0
assert indices[-1] == 899 # uniform spread reaches the final frame
# Request overrides reach the sampler through both kwarg spellings. The
# 15-frame spread is odd, so pair-padding duplicates the last frame.
assert (
len(
video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
fps=0.5,
)
)
== 16
)
assert (
len(
video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
target_fps=0.5,
)
)
== 16
)
assert (
len(
video_processor.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
max_frames=16,
)
)
== 16
)
def test_defaults_mirror_checkpoint_config():
"""Bare instantiation matches the checkpoint's ``processor_config.json``
token-budget style defaults; ``size`` carries no budget."""
image_processor = Glm5NextImageProcessor()
assert image_processor.patch_expand_factor == 1
assert image_processor.min_image_tokens == 16
assert image_processor.max_image_tokens == 8000
assert image_processor.resize_mode == "pad"
video_processor = Glm5NextVideoProcessor()
assert video_processor.patch_expand_factor == 1
assert video_processor.min_image_tokens == 16
assert video_processor.max_image_tokens == 240000
assert video_processor.fps_interval == 2.0
assert video_processor.max_frame_count_dynamic == 2048
def test_token_budgets_drive_geometry():
"""The token bounds fully determine the pixel budget and the grid."""
token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512)
img = Image.new("RGB", (400, 300))
grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist()
# Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420)
# without hitting either bound -> 22x30 patches.
assert grid == [1, 22, 30]
assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30
def test_missing_token_budgets_rejected():
proc = Glm5NextImageProcessor(min_image_tokens=None)
with pytest.raises(ValueError, match="min_image_tokens"):
proc(Image.new("RGB", (400, 300)), return_tensors="pt")
def test_video_config_fields_land():
"""fps_interval / max_frame_count_dynamic from the dedicated config
shape sampling without any request overrides."""
proc = Glm5NextVideoProcessor(
fps_interval=4,
max_frame_count_dynamic=32,
)
indices = proc.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
)
# 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32.
assert len(indices) == 32
assert indices[0] == 0
assert indices[-1] == 899
# Request overrides still win over the config values.
assert (
len(
proc.sample_frames(
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
fps=0.5,
)
)
== 16
)
+54
View File
@@ -10,6 +10,7 @@ from types import SimpleNamespace
from typing import cast
from unittest.mock import MagicMock, patch
import pytest
from transformers import PretrainedConfig
from vllm.config.model import ModelConfig
@@ -19,6 +20,59 @@ from vllm.transformers_utils.config import (
get_safetensors_params_metadata,
try_get_generation_config,
)
from vllm.transformers_utils.configs.glm5_next import (
Glm5NextConfig,
Glm5NextTextConfig,
Glm5NextVisionConfig,
)
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
layer_types = ["linear_attention", "deepseek_sparse_attention"]
config = Glm5NextTextConfig(
num_hidden_layers=len(layer_types), layer_types=layer_types
)
assert config.layer_types == layer_types
assert config.layers_block_type == ["linear_attention", "attention"]
def test_glm5_next_accepts_prebuilt_subconfigs():
text_config = Glm5NextTextConfig(hidden_size=1024)
vision_config = Glm5NextVisionConfig(hidden_size=768)
config = Glm5NextConfig(
text_config=text_config,
vision_config=vision_config,
)
assert config.text_config is text_config
assert config.vision_config is vision_config
@pytest.mark.parametrize(
("kwargs", "option"),
[
(
{"index_topk": 2048, "index_dsa_use_layernorm": False},
"index_dsa_use_layernorm",
),
(
{"index_topk": 2048, "index_kpool_compress": False},
"index_kpool_compress",
),
(
{"index_topk": 2048, "index_kpool_always_select_tail": False},
"index_kpool_always_select_tail",
),
({"hres_vwnstyle": False}, "hres_vwnstyle"),
({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"),
],
)
def test_glm5_next_rejects_unimplemented_config_options(kwargs, option):
with pytest.raises(NotImplementedError, match=option):
Glm5NextTextConfig(**kwargs)
def test_get_llama3_eos_token():
+7 -6
View File
@@ -22,7 +22,7 @@ from collections.abc import Callable, Iterable, MutableMapping, Sequence
from contextlib import ExitStack, contextmanager
from multiprocessing import Process, get_context
from pathlib import Path
from typing import Any, Literal, cast
from typing import TYPE_CHECKING, Any, Literal, cast
from unittest.mock import patch
import anthropic
@@ -46,10 +46,6 @@ from vllm.distributed import (
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.entrypoints.cli.serve import ServeSubcommand
from vllm.logger import init_logger
from vllm.model_executor.kernels.linear import (
_KernelT,
init_fp8_linear_kernel,
)
from vllm.model_executor.layers.quantization.utils.quant_utils import (
QuantKey,
)
@@ -65,6 +61,9 @@ from vllm.utils.torch_utils import (
)
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
if TYPE_CHECKING:
from vllm.model_executor.kernels.linear import _KernelT
logger = init_logger(__name__)
FP8_DTYPE = current_platform.fp8_dtype()
@@ -2349,9 +2348,11 @@ class TestFP8Layer(torch.nn.Module):
out_dtype: torch.dtype | None = None,
transpose_weights: bool = False,
device: torch.device | None = None,
force_kernel: type[_KernelT] | None = None,
force_kernel: "type[_KernelT] | None" = None,
):
super().__init__()
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
self.input_size_per_partition = weight_shape[1]
self.output_size_per_partition = weight_shape[0]
self.logical_widths = [self.output_size_per_partition]
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for error handling in the CUDA attention backend probe.
"""Tests for CUDA attention backend selection and probe error handling.
Environment-shaped probe failures (missing packages, unreadable caches,
broken driver installs) must mark the backend unavailable rather than
@@ -89,6 +89,48 @@ def test_get_valid_backends_keeps_probing_after_failure():
assert valid
def test_sm90_nope_mla_prefers_flashinfer_without_changing_rope_order():
backend_cls = MagicMock()
backend_cls.validate_configuration.return_value = []
sparse_backends = {
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
}
def sparse_order(head_size: int) -> list[AttentionBackendEnum]:
config = SELECTOR_CONFIG._replace(
head_size=head_size,
use_mla=True,
use_sparse=True,
)
with patch(
"vllm.platforms.cuda._get_attn_backend_class",
return_value=backend_cls,
):
valid, _ = CudaPlatform.get_valid_backends(
device_capability=SM90,
attn_selector_config=config,
num_heads=32,
)
return [
candidate.backend
for candidate in valid
if candidate.backend in sparse_backends
]
assert sparse_order(512) == [
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
]
assert sparse_order(576) == [
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
AttentionBackendEnum.FLASHMLA_SPARSE,
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
]
def test_selected_backend_probe_failure_raises_value_error_with_cause():
exc = OSError("libcuda.so.1: cannot open shared object file")
with (
@@ -0,0 +1,280 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU).
The FlashInfer wrapper and top-k conversion are replaced by CPU recorders;
the tests pin the contract between the impl and the kernel API: page_size=1
varlen rows, reserved-buffer refresh, plan parameters (dims, NoPE/rope scale,
causality), ckv/kpe cache splitting, and the backend's model-shape gates.
"""
from types import SimpleNamespace
import pytest
import torch
# isort: off
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 as sm90_mod
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 import (
FlashInferMLASparseSM90Backend,
FlashInferMLASparseSM90Builder,
FlashInferMLASparseSM90Impl,
)
# isort: on
BLOCK_SIZE = 64
HEAD = 512
TOPK = 128 # triton convert requires width % 128 == 0
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
out = torch.full_like(token_indices, -1)
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
for t in range(token_indices.shape[0]):
vals = []
for j in range(token_indices.shape[1]):
pos = int(token_indices[t, j])
if pos == -1:
continue
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
if blk < 0:
continue
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
counts[t] = len(vals)
return out, counts
class FakeWrapper:
def __init__(self):
self.plan_args = None
self.run_args = None
def plan(self, *args, **kwargs):
self.plan_args = (args, kwargs)
def run(self, *args, **kwargs):
q_nope, q_pe, ckv, kpe = args
self.run_args = (q_nope, q_pe, ckv, kpe, kwargs)
return torch.zeros(
q_nope.shape[0], q_nope.shape[1], ckv.shape[-1], dtype=torch.bfloat16
)
class FakeState:
def __init__(self, width, max_tokens=64):
self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32)
self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32)
self.wrapper = FakeWrapper()
self.plan_calls = []
def plan(self, num_tokens, kv_lens):
self.plan_calls.append((num_tokens, kv_lens))
def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK):
impl = object.__new__(FlashInferMLASparseSM90Impl)
impl.num_heads = num_heads
impl.head_size = HEAD + qk_rope
impl.scale = (HEAD + qk_rope) ** -0.5
impl.kv_lora_rank = HEAD
impl.qk_rope_head_dim = qk_rope
impl.kv_cache_dtype = kv_dtype
impl.use_fp8_kv_cache = kv_dtype in ("fp8", "fp8_e4m3")
rows = 4
impl.topk_indices_buffer = torch.full((rows, topk_width), -1, dtype=torch.int32)
return impl, rows
def make_batch(rows, topk_rows, own_blocks):
req_id = torch.tensor([0] * rows, dtype=torch.int32)
block_table = torch.zeros(1, 16, dtype=torch.int32)
block_table[:, 0] = own_blocks[0]
topk = torch.full((rows, TOPK), -1, dtype=torch.int32)
for t, row in enumerate(topk_rows):
topk[t, : len(row)] = torch.tensor(row, dtype=torch.int32)
return SimpleNamespace(
req_id_per_token=req_id, block_table=block_table, block_size=BLOCK_SIZE
)
@pytest.mark.parametrize("qk_rope,kv_dtype", [(0, "fp8_e4m3"), (64, "auto")])
def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
impl, rows = make_impl(qk_rope, kv_dtype)
state = FakeState(TOPK)
monkeypatch.setattr(
sm90_mod, "triton_convert_req_index_to_global_index", ref_convert
)
# req with context 10 < topk: 8 valid + -1 padding.
topk_rows = [
[7, 3, 1, 9, 0, 2, 5, 8] + [-1] * (TOPK - 8),
[4, 0, 2, 3, 1] + [-1] * (TOPK - 5),
[6, 5, 4] + [-1] * (TOPK - 3),
[2] + [-1] * (TOPK - 1),
]
meta = make_batch(rows, topk_rows, [3])
meta.state = state
q_nope = torch.randn(rows, impl.num_heads, HEAD)
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
cache = torch.zeros(
8 * BLOCK_SIZE,
impl.head_size,
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
)
out, lse = impl.forward_mqa(
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
)
assert lse is None and out.shape == (rows, impl.num_heads, HEAD)
# Reserved buffers carry this step's slots; lengths are NOT refreshed
# here (the builder plans them host-side before capture/replay).
ref_slots, ref_counts = ref_convert(
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
)
width = TOPK
got_slots = state.kv_indices[: rows * width].view(rows, width)
for t in range(rows):
k = int(ref_counts[t])
assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist()
assert state.plan_calls == []
assert state.wrapper.run_args is not None
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
assert kpe.shape[-1] == qk_rope
if impl.use_fp8_kv_cache:
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
else:
assert kwargs == {}
def test_builder_attaches_its_state(monkeypatch):
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
builder.state = FakeState(TOPK)
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
metadata.state = None
monkeypatch.setattr(
sm90_mod.FlashInferMLASparseMetadataBuilder,
"build",
lambda *_args, **_kwargs: metadata,
)
cam = SimpleNamespace(
num_reqs=1,
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
seq_lens=torch.tensor([1], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
positions=None,
)
result = builder.build(0, cam)
assert result.state is builder.state
assert builder.state.plan_calls[0][0] == 1
assert builder.state.plan_calls[0][1].tolist() == [1]
def test_plan_uses_state_params(monkeypatch):
"""The NoPE/rope dims and scale live on the builder state, not the layer.
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
call (contexts grow between steps) and the indptrs are always full-size
with zero-query padding rows past num_tokens.
"""
impl, rows = make_impl(64, "auto")
wrapper = FakeWrapper()
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
state.device = torch.device("cpu")
state.wrapper = wrapper
state.num_heads = 4
state.kv_dtype = torch.bfloat16
state.kv_lora_rank = HEAD
state.qk_rope_head_dim = 64
state.sm_scale = 576**-0.5
state.max_tokens = 4
state.topk_width = TOPK
state.kv_indices = torch.zeros(4 * TOPK)
state._arange_cpu = torch.arange(5, dtype=torch.int32)
state._qo_cpu = torch.empty(5, dtype=torch.int32)
state._kv_cpu = torch.empty(5, dtype=torch.int32)
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
assert wrapper.plan_args is not None
args, kwargs = wrapper.plan_args
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries
assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)]
assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
assert scale == 576**-0.5
assert kwargs["q_data_type"] == torch.bfloat16
assert kwargs["kv_data_type"] == torch.bfloat16
def test_kv_lens_host_formula():
"""Per-row host lengths: context == position + 1; capped at
index_topk + trailing-pool remainder past the sparse threshold."""
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
builder._async_scheduling = False
cam = SimpleNamespace(
num_reqs=3,
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
positions=None,
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 10
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
# req1: positions 7,8 -> ctx 8,9
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
def test_kv_lens_host_empty():
builder = object.__new__(FlashInferMLASparseSM90Builder)
builder._index_topk = 2048
builder._index_kpool = 4
cam = SimpleNamespace(
num_reqs=0,
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
seq_lens=torch.zeros(0, dtype=torch.int32),
)
num_rows, lens = builder._kv_lens_host(cam)
assert num_rows == 0 and lens.numel() == 0
def test_supports_combination_gates(monkeypatch, default_vllm_config):
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
head_size=576,
dtype=torch.bfloat16,
kv_cache_dtype="fp8_e4m3",
block_size=64,
use_mla=True,
has_sink=False,
use_sparse=True,
use_mm_prefix=False,
device_capability=SimpleNamespace(major=9),
**kw,
)
assert call() is None # no model config: only the feature gate applies
import vllm.config as cfg
monkeypatch.setattr(
cfg,
"get_current_vllm_config",
lambda: SimpleNamespace(model_config=None),
)
assert call() is None
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: False)
assert "requires FlashInfer" in (call() or "")
@@ -0,0 +1,369 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for the kpool tail slot mapping (no GPU required).
The kpool tail cache is a 1-block-per-request circular ring addressed by
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
allocated per request, never grown, so only column 0 of its block table is
ever written; the rest stays zero-initialized).
The generic per-group slot kernel cannot express that layout: it maps
``pos -> bt[req][pos // bs] * bs + pos % bs`` (``_compute_slot_mappings_kernel``
in vllm/v1/worker/gpu/block_table.py), so every token at ``pos >= kpool``
reads a zero column and collapses onto physical tail block 0. All concurrent
requests then share one ``kpool``-slot ring and corrupt each other's pool
compression.
These tests pin that defect's arithmetic, verify the circular replacement
(``compute_kpool_tail_slot_mapping``), and mirror the tail kernels' index
math to show cross-request pollution before the fix and isolation after.
"""
from types import SimpleNamespace
import pytest
import torch
from vllm.v1.attention.backend import CommonAttentionMetadata
from vllm.v1.attention.backends.mla.indexer import (
KpoolTailBackend,
KpoolTailMetadataBuilder,
compute_kpool_tail_slot_mapping,
)
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
from vllm.v1.kv_cache_layout import KVCacheLayout
KPOOL = 4
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
spec = KpoolTailSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
_, _, head_stride, state_stride, content_stride = strides
assert layout is KVCacheLayout.LBHNC
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
assert state_stride == 128 * torch.bfloat16.itemsize
assert content_stride == 1
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
written and stay zero."""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
return bt
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
(block_table.py:305-313) applied to the tail group's table."""
slots = []
for req in range(block_table.shape[0]):
for i in range(query_start_loc[req], query_start_loc[req + 1]):
pos = int(positions[i])
block_number = int(block_table[req, pos // KPOOL])
slots.append(block_number * KPOOL + pos % KPOOL)
return torch.tensor(slots, dtype=torch.int64)
def circular_tail_slots(
slot_mapping, block_table, query_start_loc, positions, num_actual, num_reqs
):
return compute_kpool_tail_slot_mapping(
slot_mapping,
block_table,
query_start_loc,
positions,
num_actual,
num_reqs,
KPOOL,
)
def make_batch(per_req_positions, padded_len=None):
positions = torch.cat(
[torch.tensor(p, dtype=torch.int64) for p in per_req_positions]
)
num_actual = positions.numel()
num_reqs = len(per_req_positions)
lens = [len(p) for p in per_req_positions]
qsl = torch.zeros(num_reqs + 1, dtype=torch.int64)
torch.cumsum(torch.tensor(lens, dtype=torch.int64), 0, out=qsl[1:])
if padded_len is None:
padded_len = num_actual
slot_mapping = torch.full((padded_len,), -1, dtype=torch.int64)
return positions, qsl, slot_mapping, num_actual, num_reqs
def test_legacy_generic_mapping_collapses_onto_block_zero():
"""The bug: with the manager's 1-column block table, the generic kernel
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
bt = make_tail_block_table(own_blocks)
legacy = legacy_generic_tail_slots(bt, qsl, positions)
# Every token at pos >= kpool resolves to block 0, not the request's own.
off = 0
for req, prompt in enumerate(per_req):
req_slots = legacy[off : off + len(prompt)]
for pos in range(len(prompt)):
slot = int(req_slots[pos])
if pos >= KPOOL:
assert slot // KPOOL == 0, (
f"expected collapse onto block 0 at pos {pos}"
)
assert slot // KPOOL != own_blocks[req]
off += len(prompt)
# The two requests share ring slots -> cross-request pollution.
a_slots = set(legacy[: len(per_req[0])].tolist())
b_slots = set(legacy[len(per_req[0]) :].tolist())
assert a_slots & b_slots, "legacy mapping must collide across requests"
def test_circular_mapping_isolates_requests():
"""The fix: every token lands in its own request's block at pos % kpool,
and no slot is ever shared by two different requests (slots do recur
within a request every kpool positions -- that is the circular design)."""
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
bt = make_tail_block_table(own_blocks)
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
off = 0
per_req_slots = []
for req, prompt in enumerate(per_req):
req_slots = set()
for pos in range(len(prompt)):
slot = int(out[off + pos])
assert slot // KPOOL == own_blocks[req], (
f"req {req} pos {pos} left its tail block"
)
assert slot % KPOOL == pos % KPOOL
req_slots.add(slot)
per_req_slots.append(req_slots)
off += len(prompt)
assert not per_req_slots[0] & per_req_slots[1]
@pytest.mark.parametrize("prompt_len", [1, 2, 3, 4])
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
"""For pos < kpool the generic kernel already picks the own block, so the
two mappings agree while every position fits the request's first block
(single-request behavior is unchanged)."""
own_blocks = [7]
per_req = [list(range(prompt_len))]
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
bt = make_tail_block_table(own_blocks)
legacy = legacy_generic_tail_slots(bt, qsl, positions)
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
assert torch.equal(out, legacy)
def test_circular_mapping_preserves_padding_and_empty_batch():
own_blocks = [5, 9]
per_req = [list(range(10)), list(range(12))]
padded_len = sum(len(p) for p in per_req) + 8
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
per_req, padded_len=padded_len
)
bt = make_tail_block_table(own_blocks)
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
assert out.shape == slot_mapping.shape
assert torch.equal(out[num_actual:], torch.full_like(out[num_actual:], -1))
empty = circular_tail_slots(slot_mapping, bt, qsl, positions[:0], 0, num_reqs)
assert torch.equal(empty, slot_mapping)
def make_common_metadata(per_req_positions, own_blocks, with_positions=True):
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
per_req_positions, padded_len=sum(len(p) for p in per_req_positions) + 4
)
bt = make_tail_block_table(own_blocks)
seq_lens = torch.tensor(
[max(p) + 1 if p else 1 for p in per_req_positions], dtype=torch.int64
)
return CommonAttentionMetadata(
query_start_loc=qsl,
query_start_loc_cpu=qsl.clone(),
seq_lens=seq_lens,
num_reqs=num_reqs,
num_actual_tokens=num_actual,
max_query_len=max((len(p) for p in per_req_positions), default=1),
max_seq_len=int(seq_lens.max()) if num_reqs else 1,
block_table_tensor=bt,
slot_mapping=slot_mapping,
positions=positions if with_positions else None,
)
def make_tail_builder(block_size=KPOOL, max_num_batched_tokens=128):
builder = object.__new__(KpoolTailMetadataBuilder)
builder.kv_cache_spec = SimpleNamespace(block_size=block_size)
builder.slot_mapping_buffer = torch.empty(max_num_batched_tokens, dtype=torch.int64)
return builder
def test_builder_build_uses_circular_mapping():
per_req = [list(range(10)), list(range(12))]
own_blocks = [5, 9]
cam = make_common_metadata(per_req, own_blocks)
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
out = meta.slot_mapping
off = 0
for req, prompt in enumerate(per_req):
for pos in range(len(prompt)):
slot = int(out[off + pos])
assert slot // KPOOL == own_blocks[req]
assert slot % KPOOL == pos % KPOOL
off += len(prompt)
# Padding tail of the buffer keeps the -1 sentinel.
assert torch.equal(
out[cam.num_actual_tokens :], torch.full_like(out[cam.num_actual_tokens :], -1)
)
def test_builder_build_falls_back_without_positions():
"""Capture / dummy builds without positions keep the generic mapping."""
per_req = [list(range(10))]
cam = make_common_metadata(per_req, [5], with_positions=False)
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
assert meta.slot_mapping is cam.slot_mapping
def test_builder_reuses_slot_mapping_storage():
builder = make_tail_builder()
first = make_common_metadata([list(range(10))], [5])
first_meta = KpoolTailMetadataBuilder.build(builder, 0, first)
data_ptr = first_meta.slot_mapping.data_ptr()
second = make_common_metadata([list(range(12))], [9])
second_meta = KpoolTailMetadataBuilder.build(builder, 0, second)
assert second_meta.slot_mapping.data_ptr() == data_ptr
assert second_meta.slot_mapping[:12].tolist() == [
9 * KPOOL + pos % KPOOL for pos in range(12)
]
# ---------------------------------------------------------------------------
# Index-level mirror of the tail kernels: seed / stash / pool completion
# (addressing replicated from kpool_compress.py's Triton kernels).
# ---------------------------------------------------------------------------
class TailRingMirror:
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
completing at pos reads ring slots (pool_start + s) % kpool and uses the
current token's own K/score for the last member."""
def __init__(self, num_blocks, kpool=KPOOL):
self.kpool = kpool
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
def stash(self, tail_slot, pos, k, s):
blk, off = tail_slot // self.kpool, pos % self.kpool
self.k[blk, off] = k
self.s[blk, off] = s
seed = stash # the seed kernel writes with the same addressing
def complete(self, tail_slot, pos, k, s):
blk = tail_slot // self.kpool
start = pos - (self.kpool - 1)
kk = torch.stack(
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
)
ss = torch.stack(
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
)
kk[-1], ss[-1] = k, s # is_current for the completing token
w = torch.softmax(ss, dim=0)
return (kk * w).sum(0)
def token_kv(req, pos):
k = torch.tensor([pos + 100.0 * req, pos + 0.5, 2.0 * pos + 0.25])
s = torch.tensor([0.1 * (pos + 1) + req, 0.2 * pos, 0.05 * pos])
return k, s
def tail_slot_for(mapping, req, pos, own_block):
if mapping == "legacy":
bt_val = own_block if pos < KPOOL else 0
return bt_val * KPOOL + pos % KPOOL
return own_block * KPOOL + pos % KPOOL
def run_scenario(mapping, interleave):
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
ring = TailRingMirror(num_blocks=16)
blocks = {"A": 5, "B": 9}
prompts = {"A": 9, "B": 11}
def slot(req, pos):
return tail_slot_for(mapping, 0 if req == "A" else 1, pos, blocks[req])
# Prefill: seed each request's trailing incomplete pool.
for req, L in prompts.items():
for pos in range(L - (L % KPOOL or KPOOL), L):
if pos < 0:
continue
ring.stash(slot(req, pos), pos, *token_kv(0 if req == "A" else 1, pos))
# Decode: A emits pos 9, 10, 11; B emits 11, 12, 13. `interleave`
# processes B before A within a step, which is exactly what concurrent
# Triton programs do when they share tail block 0.
order = ["B", "A"] if interleave else ["A", "B"]
decode = {"A": [9, 10, 11], "B": [11, 12, 13]}
result = None
for step in range(3):
for req in order:
pos = decode[req][step]
k, s = token_kv(0 if req == "A" else 1, pos)
if pos % KPOOL == KPOOL - 1:
pool = ring.complete(slot(req, pos), pos, k, s)
if req == "A":
result = pool
ring.stash(slot(req, pos), pos, k, s)
return result
def test_interleaved_decode_pollution_legacy_vs_circular():
"""Ground truth: request A decoding alone (its ring used exclusively)."""
torch.manual_seed(0)
ground_truth = run_scenario("circular", interleave=False)
legacy = run_scenario("legacy", interleave=True)
circular = run_scenario("circular", interleave=True)
# The old mapping lets request B's tokens into A's ring: A's boundary
# pool is compressed from 2 of B's tokens -> wrong.
assert not torch.allclose(legacy, ground_truth), (
f"legacy mapping unexpectedly clean: {legacy} vs {ground_truth}"
)
# The circular mapping keeps the rings isolated under interleaving.
torch.testing.assert_close(circular, ground_truth)
@@ -0,0 +1,127 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import torch
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
_use_rocm_sparse_triton,
fit_kpool_indices_to_aiter,
)
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_sparse_kv_row_offset,
_validate_dsv4_sparse_dims,
_validate_sparse_dims,
)
@triton.jit
def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexpr):
slot = tl.load(slot_ptr)
tl.store(output_ptr, _sparse_kv_row_offset(slot, stride))
def test_fit_kpool_indices_preserves_tail_and_best_history():
token_indices = torch.tensor(
[
[10, 9, 8, 7, 6, 5, 100, 101],
[10, 9, 8, -1, -1, -1, 100, -1],
[-1, -1, -1, -1, -1, -1, -1, -1],
],
dtype=torch.int32,
)
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6)
assert fitted.tolist() == [
[10, 9, 8, 7, 100, 101],
[10, 9, 8, 100, -1, -1],
[-1, -1, -1, -1, -1, -1],
]
def test_fit_kpool_indices_exact_width_is_noop():
token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32)
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=4)
assert fitted.data_ptr() == token_indices.data_ptr()
def test_fit_kpool_indices_rejects_narrow_input():
with pytest.raises(ValueError, match="at least topk_tokens"):
fit_kpool_indices_to_aiter(
torch.zeros((1, 3), dtype=torch.int32), topk_tokens=4
)
@pytest.mark.parametrize(
(
"kv_cache_dtype",
"head_size",
"num_prefills",
"num_decodes",
"num_decode_tokens",
"max_query_len",
"expected",
),
[
("auto", 512, 1, 0, 0, 32, True),
("auto", 512, 1, 2, 2, 32, True),
("auto", 512, 0, 2, 2, 1, True),
("fp8", 512, 1, 0, 0, 32, False),
("auto", 576, 1, 0, 0, 32, False),
("auto", 512, 0, 2, 4, 2, False),
],
)
def test_rocm_sparse_triton_route(
kv_cache_dtype,
head_size,
num_prefills,
num_decodes,
num_decode_tokens,
max_query_len,
expected,
):
assert (
_use_rocm_sparse_triton(
kv_cache_dtype=kv_cache_dtype,
head_size=head_size,
kv_lora_rank=512,
num_prefills=num_prefills,
num_decodes=num_decodes,
num_decode_tokens=num_decode_tokens,
max_query_len=max_query_len,
)
is expected
)
def test_rocm_sparse_attention_accepts_glm_nope_dimensions():
_validate_sparse_dims(512, 512, 0, "test")
def test_rocm_sparse_attention_rejects_inconsistent_dimensions():
with pytest.raises(AssertionError, match="expected head_dim"):
_validate_sparse_dims(511, 512, 0, "test")
def test_dsv4_sparse_attention_keeps_layout_constraint():
_validate_dsv4_sparse_dims(512, 448, 64, "test")
with pytest.raises(AssertionError, match="expects 448 NoPE dims"):
_validate_dsv4_sparse_dims(512, 512, 0, "test")
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
def test_sparse_prefill_kv_row_offset_does_not_overflow_int32():
# GLM's 640-token pages cross the signed-int32 address boundary at block
# 6554 for a 512-element KV row. The production kernel must promote the
# slot before multiplying by the row stride.
slot = torch.tensor([6554 * 640], dtype=torch.int32, device="cuda")
output = torch.empty(1, dtype=torch.int64, device="cuda")
_store_sparse_kv_row_offset_kernel[(1,)](slot, output, stride=512)
assert output.item() == 6554 * 640 * 512
@@ -0,0 +1,184 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for the decode top-k row->token mapping (no GPU required).
On non-uniform decode batches (``requires_padding`` -- mixed plain-decode and
spec-verify requests, or variable MTP verify lens; taken on Hopper where the
varlen/flatten logits path is unavailable), the pool-topk rows follow the
PADDED ``[batch_size, next_n]`` grid: row ``(b, t)`` is flat decode token
``offset_b + t``. The former inline ``dec_seq = positions[:n] + 1``
(``n = batch_size * next_n``) indexes the flat per-token layout with padded
coordinates, so rows after the first non-uniform request read another
request's positions and rows past the decode region read prefill tokens --
``expand_pools_and_append_tail`` then anchors the tail at a foreign length.
These tests pin the defect with the exact production arithmetic and verify
the layout-aware replacement (``_decode_topk_seq_lens``), including the tail
expansion consequences via the pure-torch expand/append pair that the fused
kernel is documented to replicate.
"""
import torch
# Bootstrap the glm5next package before entering the indexer module: its
# kpool_compress import runs glm5next/__init__, which pulls model ->
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
# class at module scope). Production always enters via attention.py first.
# isort: off
import vllm.models.glm5next # noqa: F401
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
append_tail_to_topk,
expand_pools_to_tokens,
)
# isort: on
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
_decode_topk_seq_lens,
_fill_short_decode_causal_indices,
)
from vllm.platforms import current_platform
KPOOL = 4
TOPK_TOKENS = 16
SELECT_K = TOPK_TOKENS // KPOOL
def test_kpool_ops_dispatch_matches_platform():
expected_backend = ".amd." if current_platform.is_rocm() else ".nvidia."
assert expected_backend in indexer_mod.kpool_ops.__name__
def test_short_decode_fills_exact_causal_rows():
topk = torch.full((3, 8), 99, dtype=torch.int32)
positions = torch.tensor([0, 3, 7], dtype=torch.int64)
assert _fill_short_decode_causal_indices(topk, positions, 3, 8, 8)
assert topk.tolist() == [
[0, -1, -1, -1, -1, -1, -1, -1],
[0, 1, 2, 3, -1, -1, -1, -1],
[0, 1, 2, 3, 4, 5, 6, 7],
]
def test_short_decode_leaves_buffer_unchanged_for_sparse_context():
topk = torch.full((2, 8), 99, dtype=torch.int32)
before = topk.clone()
assert not _fill_short_decode_causal_indices(topk, torch.tensor([7, 8]), 2, 9, 8)
assert torch.equal(topk, before)
def make_non_uniform_batch():
"""3 requests: plain decode (1 token), MTP verify (4), adaptive verify (3).
Flat decode positions (production layout: decode tokens first, then any
prefill tokens of the same batch):
req0: [30] (context len 30)
req1: [100..103] (verify at context len 100)
req2: [7, 8, 9] (verify at context len 7)
Followed by 4 prefill tokens at positions 555..558 so the flat tensor is
at least ``n = 3 * 4`` long, as it is in a real mixed batch.
"""
decode_lens = torch.tensor([1, 4, 3], dtype=torch.int64)
per_req_positions = [[30], [100, 101, 102, 103], [7, 8, 9]]
flat_decode = [p for req in per_req_positions for p in req]
positions = torch.tensor(flat_decode + [555, 556, 557, 558], dtype=torch.int64)
return decode_lens, per_req_positions, positions
def expected_row_seq_lens(per_req_positions, batch_size, next_n):
"""Ground truth: row (b, t) -> pos + 1 for real rows, 0 for pad rows."""
out = torch.zeros(batch_size * next_n, dtype=torch.int32)
for b, req in enumerate(per_req_positions):
for t, pos in enumerate(req):
out[b * next_n + t] = pos + 1
return out
def test_legacy_flat_layout_misaligns_and_bleeds():
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
requests' positions and prefill positions."""
decode_lens, per_req, positions = make_non_uniform_batch()
batch_size = decode_lens.shape[0]
next_n = int(decode_lens.max())
n = batch_size * next_n
assert n == 12
legacy = positions[:n].to(torch.int32) + 1
expected = expected_row_seq_lens(per_req, batch_size, next_n)
# Row (1, 0): req1's first verify token (pos 100, seq 101) reads flat
# index 4 -> req1's LAST verify token (pos 103). Its true tail token is
# dropped and the tail anchors 3 tokens late.
assert int(legacy[4]) == 104 and int(expected[4]) == 101
# Rows (2, *): req2 starts at flat offset 5, but padded coordinates point
# at flat indices 8..11 -- the batch's PREFILL positions.
for t in range(3):
assert int(legacy[2 * next_n + t]) >= 556, (
"legacy row (2, t) should read prefill positions"
)
assert not torch.equal(legacy, expected)
def test_helper_uniform_layout_matches_flat_slice():
"""Uniform batches keep the flat shortcut (zero behavior/perf change)."""
per_req = [[200, 201, 202, 203], [50, 51, 52, 53]]
positions = torch.tensor([p for r in per_req for p in r], dtype=torch.int64)
decode_lens = torch.tensor([4, 4], dtype=torch.int64)
out = _decode_topk_seq_lens(positions, decode_lens, 8, 2, 4, requires_padding=False)
assert torch.equal(out, positions[:8].to(torch.int32) + 1)
def test_helper_padded_layout_per_row():
"""Non-uniform batches map every padded row to its own token's position;
pad rows get 0 (empty tail)."""
decode_lens, per_req, positions = make_non_uniform_batch()
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
expected = expected_row_seq_lens(per_req, 3, 4)
assert torch.equal(out, expected)
# Pad rows (0, 1..3) and (2, 3) collapse to 0 -> no tail appended.
assert out[1] == 0 and out[2] == 0 and out[3] == 0 and out[11] == 0
def expand_tail_region(dec_seq):
"""Run the production pure-torch expand + append pair (the fused kernel
replicates it exactly on the identity path) and return the tail columns
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
rows = dec_seq.shape[0]
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
valid = torch.ones_like(pool_ids, dtype=torch.bool)
expanded = expand_pools_to_tokens(pool_ids, valid, TOPK_TOKENS, KPOOL)
seq_lens = dec_seq.to(torch.int32)
pool_lens = (seq_lens // KPOOL).to(torch.int32)
out = append_tail_to_topk(expanded, seq_lens, pool_lens, KPOOL)
return out[:, TOPK_TOKENS:]
def test_tail_expansion_legacy_vs_fixed():
"""End-to-end tail consequence: the fixed mapping appends exactly the
request's trailing incomplete pool; the legacy flat slice drops real tail
tokens and emits indices far past the request's own sequence."""
decode_lens, per_req, positions = make_non_uniform_batch()
n = 12
fixed = _decode_topk_seq_lens(
positions, decode_lens, 8, 3, 4, requires_padding=True
)
legacy = positions[:n].to(torch.int32) + 1
fixed_tail = expand_tail_region(fixed)
legacy_tail = expand_tail_region(legacy)
# Fixed: row (1, 0) (seq 101) keeps its single tail token 100; row (2, 2)
# (seq 10) keeps its full trailing pool [8, 9].
assert fixed_tail[4, 0].item() == 100
assert fixed_tail[10, :2].tolist() == [8, 9]
# Legacy: row (1, 0) loses its tail token (anchored at 104, count 0)...
assert legacy_tail[4, 0].item() == -1
# ...and row (2, 2) reads prefill position 557 -> tail indices 556/557,
# way past req2's 10-token sequence -> out-of-bounds block-table reads.
assert legacy_tail[10, :2].tolist() == [556, 557]
assert not torch.equal(legacy_tail, fixed_tail)
+61 -5
View File
@@ -40,11 +40,13 @@ if not current_platform.is_cuda():
allow_module_level=True,
)
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse_mod
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
from vllm.utils.math_utils import cdiv
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseImpl,
FlashInferMLASparseTRTLLMBackend,
)
from vllm.v1.attention.backends.mla.flashmla_sparse import (
@@ -82,6 +84,60 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
DEVICE_TYPE = current_platform.device_type
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
"""Weight absorption must not change the model's attention temperature."""
model_scale = 256**-0.5
kv_lora_rank = 512
topk = torch.zeros((1, 1), dtype=torch.int32)
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(1, dtype=torch.int32),
block_table=torch.zeros((1, 1), dtype=torch.int32),
block_size=1,
)
recorded_scale = None
impl = object.__new__(FlashInferMLASparseImpl)
impl.scale = model_scale
impl.qk_nope_head_dim = 256
impl.kv_lora_rank = kv_lora_rank
impl.qk_rope_head_dim = 0
impl.kv_cache_dtype = "auto"
impl.topk_indices_buffer = topk
impl.dcp_world_size = 1
impl._workspace_buffer = torch.empty(1)
impl.bmm1_scale = None
impl.bmm2_scale = None
impl.is_nope_mla = True
impl.need_to_return_lse_for_decode = False
monkeypatch.setattr(
flashinfer_sparse_mod,
"triton_convert_req_index_to_global_index",
lambda *args, **kwargs: (topk, torch.ones(1, dtype=torch.int32)),
)
import flashinfer.decode
def fake_flashinfer(**kwargs):
nonlocal recorded_scale
recorded_scale = kwargs["bmm1_scale"]
return torch.zeros((1, 1, 1, kv_lora_rank))
monkeypatch.setattr(
flashinfer.decode,
"trtllm_batch_decode_with_kv_cache_mla",
fake_flashinfer,
)
impl.forward_mqa(
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 1, kv_lora_rank),
metadata,
SimpleNamespace(),
)
assert recorded_scale == model_scale
assert recorded_scale != kv_lora_rank**-0.5
def _float_to_e8m0_truncate(f: float) -> float:
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
e8m0 format only stores the exponent (power of 2).
@@ -1451,14 +1507,14 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
return_valid_counts=False,
)
assert isinstance(result_only, torch.Tensor)
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
for row, num_valid in enumerate(expected_valid):
compact_valid = result[row, :num_valid].sort().values
original_valid = result_only[row][result_only[row] >= 0].sort().values
torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
assert torch.all(result[row, num_valid:] == -1)
def test_flashmla_cache_dtype_aliases_use_ds_layout():
from vllm.model_executor.layers.attention.mla_attention import (
_canonicalize_sparse_mla_kv_cache_dtype,
)
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
# stores kv_cache_dtype, so they cannot bypass the gate.
for alias in ("fp8", "fp8_e4m3"):
@@ -23,6 +23,7 @@ from vllm.v1.core.kv_cache_utils import (
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
@@ -1801,6 +1802,127 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
def test_opted_out_scratch_group_keeps_partial_hash_hits():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
block_size=mamba_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=mamba_block_size,
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
assert manager.coordinator.enable_partial_hash_hits
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
manager.new_step_starts()
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == 6
def test_kpool_tail_supports_128_token_partial_hash_hits():
hash_block_size = 128
cache_block_size = 9 * hash_block_size
kpool_block_size = 64
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=cache_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=cache_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
block_size=kpool_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=kpool_block_size,
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
assert manager.coordinator.scheduler_block_size == cache_block_size
assert manager.coordinator.hash_block_size == hash_block_size
assert manager.coordinator.enable_partial_hash_hits
shared_prefix = [10] * (12 * hash_block_size)
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
assert (
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
is not None
)
manager.free(req0)
manager.new_step_starts()
req1 = make_request("1", shared_prefix + [12] * 64, hash_block_size, sha256)
_, num_computed, _ = manager.get_computed_blocks(req1)
assert num_computed == len(shared_prefix)
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
hash_block_size = 2
+570 -29
View File
@@ -6,13 +6,19 @@ import importlib
from collections.abc import Callable
from dataclasses import replace
from types import SimpleNamespace
from typing import Any
from typing import Any, cast
import pytest
import torch
import vllm.v1.core.kv_cache_utils as kv_cache_utils
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
from vllm.config import (
CacheConfig,
KVTransferConfig,
ModelConfig,
SchedulerConfig,
VllmConfig,
)
from vllm.config.kv_events import KVEventsConfig
from vllm.lora.request import LoRARequest
from vllm.multimodal.inputs import (
@@ -47,6 +53,7 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
FullAttentionSpec,
HiddenStateCacheSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
@@ -2191,34 +2198,567 @@ def test_generate_scheduler_kv_cache_config():
)
def test_mixed_precision_kv_cache_with_uniform_type_specs():
fp8_spec = new_kv_cache_spec(dtype=torch.float8_e4m3fn)
bf16_spec = new_kv_cache_spec(dtype=torch.bfloat16)
worker_config = KVCacheConfig(
num_blocks=10,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["fp8_layer"],
UniformTypeKVCacheSpecs(
block_size=16, kv_cache_specs={"fp8_layer": fp8_spec}
),
),
KVCacheGroupSpec(
["bf16_layer"],
UniformTypeKVCacheSpecs(
block_size=16, kv_cache_specs={"bf16_layer": bf16_spec}
),
),
],
def _glm5_like_kv_cache_spec(
mamba_spec_factory=new_mamba_spec,
) -> tuple[dict[str, KVCacheSpec], list[str]]:
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
kv_cache_spec: dict[str, KVCacheSpec] = {}
mamba_layers = []
for i in range(45):
if i % 4 == 3:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=16,
)
else:
name = f"layers.{i}.linear_attn"
kv_cache_spec[name] = mamba_spec_factory()
mamba_layers.append(name)
return kv_cache_spec, mamba_layers
def _glm5_like_kv_cache_spec_with_tail(
mamba_spec_factory=new_mamba_spec,
) -> dict[str, KVCacheSpec]:
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
is why the fixture drops the tokens_per_state=16 shape used above.
"""
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
for i in range(3, 45, 4):
kpool = 4
kv_cache_spec[f"layers.{i}.indexer"] = replace(
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
tokens_per_state=kpool,
)
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
block_size=kpool,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=kpool,
)
return kv_cache_spec
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
return {
layer_name: tensor
for tensor in kv_cache_config.kv_cache_tensors
for layer_name in tensor.layers
}
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
def test_get_kv_cache_config_balanced_mamba_hybrid():
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
uniform_groups = [
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
]
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
# gets an MLA slot.
assert len(groups) == 5
assert len(uniform_groups) == 1
assert len(uniform_groups[0].layer_names) == 22
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
for k, name in enumerate(mamba_layers):
assert name in mamba_groups[k % 4].layer_names
# Mamba pages are padded up to the MLA page.
for group in mamba_groups:
assert group.kv_cache_spec.page_size_padded == mla_page
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
# Every block id is charged the full per-block sum.
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
assert (
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
== blocks_per_request * bytes_per_block
)
scheduler_config = generate_scheduler_kv_cache_config([worker_config])
assert worker_config.needs_kv_cache_zeroing
assert scheduler_config.needs_kv_cache_zeroing
available_memory = bytes_per_block * 100 + 1
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, available_memory
)
assert kv_cache_config.num_blocks == 100
# Every logical layer has a view into one backing allocation. Mamba views
# alias their corresponding MLA slot by using the same byte offset.
assert len(kv_cache_config.kv_cache_tensors) == 56
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
tensors = _tensor_by_layer(kv_cache_config)
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
for i, mla_name in enumerate(mla_layer_names):
mla_tensor = tensors[mla_name]
assert mla_tensor.block_stride == mla_page
for group in mamba_groups:
if i < len(group.layer_names):
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
for i in range(11):
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
assert total_allocated == bytes_per_block * 100
assert 0 <= available_memory - total_allocated < bytes_per_block
assert get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config
) == pytest.approx(100 / blocks_per_request)
def new_mla_spec(cache_dtype_str=None, block_size=16):
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
"""The kpool tail parasitizes the indexer tensors instead of getting its
own: sibling idx/tail tensors paired by layer order, zero standalone tail
bytes, one shared block per request, and no prefix-caching leakage from
the kpool-sized scratch group."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
assert tail_logical_page < idx_page
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
tail_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
and all(
isinstance(spec, KpoolTailSpec)
for spec in group.kv_cache_spec.kv_cache_specs.values()
)
)
# The tail never prefix-caches; its page is padded up to the indexer page
# so the runner's strided view rides the indexer storage.
assert not tail_group.kv_cache_spec.prefix_cacheable
tail_inner = cast(
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
)
assert tail_inner.page_size_padded == idx_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
# Tail layers get logical views but no additional storage: each view aliases
# its sibling indexer at the same offset in the shared allocation.
assert len(kv_cache_config.kv_cache_tensors) == 67
tensors = _tensor_by_layer(kv_cache_config)
for i in range(11):
idx_name = f"layers.{4 * i + 3}.indexer"
tail_name = f"layers.{4 * i + 3}.tail"
assert tensors[idx_name].offset == tensors[tail_name].offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
# The layout detector surfaces the sibling names in layer order for the
# accounting and connector paths.
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
assert layout is not None
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
# Accounting charges exactly one extra shared block per request for the
# tail, not a per-sequence allocation.
attn_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
and group is not tail_group
)
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
assert (
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
== blocks_per_request * bytes_per_block
)
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
prefix-cache hash granularity: participating groups alone decide it."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
def align_mamba():
return new_mamba_spec(mamba_cache_mode="align")
groups = kv_cache_utils.get_kv_cache_groups(
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
hash_vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
kv_transfer_config=object(),
)
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
# the lcm including the tail (1024 % 4 == 0, so it coincides).
assert kv_cache_utils.resolve_kv_cache_block_sizes(
kv_cache_config, hash_vllm_config
) == (1024, 16)
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
def big_mamba_spec():
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert big_mamba_spec().page_size_bytes > mla_page
with pytest.raises(ValueError, match="does not fit the MLA page"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
"""Use the generic layout error when no kpool indexer is present."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBHNC"
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(27):
if i % 4 == 3 or i == 26:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
shapes=((512, 1024),), dtypes=(torch.float32,)
)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
with pytest.raises(NotImplementedError, match="page size"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
disable slot sharing; the layout re-pads them to the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
def prepadded_mamba_spec():
return new_mamba_spec(page_size_padded=294_912)
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert prepadded_mamba_spec().page_size_bytes < mla_page
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
assert len(groups) == 5
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
for group in mamba_groups:
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
assert len(kv_cache_config.kv_cache_tensors) == 56
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
"""Round-robin mamba grouping keeps practical PP splits balanced: every
stage's largest projected mamba group slice fits its projected MLA
layers, so per-stage slot tensors all have an MLA owner."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
# PP=2 split at transformer layer 22/23.
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
# layers, projected mamba groups [5, 5, 4, 4].
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
global_groups, worker_spec
)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 5 * mla_page + 5 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
tensors = _tensor_by_layer(kv_cache_config)
# Every projected Mamba slot aliases an MLA owner.
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
# The last slot only hosts the two groups with a 5th projected layer.
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
grouping bumps to 5 groups so every stage's projection keeps sharing
on instead of silently falling back on stage 0."""
from vllm.config import ParallelConfig
from vllm.distributed.utils import get_pp_indices
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=8192),
parallel_config=ParallelConfig(pipeline_parallel_size=4),
)
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
for k, name in enumerate(mamba_layers):
assert name in mamba_groups[k % 5].layer_names
for rank in range(4):
start, end = get_pp_indices(45, rank, 4)
worker_spec = {
n: s
for n, s in kv_cache_spec.items()
if start <= int(n.split(".")[1]) < end
}
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
groups, worker_spec
)
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
from vllm.config import ParallelConfig
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=8192),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
"""Hybrid slot sharing must take precedence over the experimental
enable_cross_layers_blocks packed layout: generic packing would give the
MLA slots a strided view, breaking the contiguous virtual split."""
model_config = ModelConfig(max_model_len=8192)
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
)
vllm_config = VllmConfig(
model_config=model_config, kv_transfer_config=kv_transfer_config
)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
assert len(kv_cache_config.kv_cache_tensors) == 56
assert all(
tensor.block_stride in (mla_page, idx_page)
for tensor in kv_cache_config.kv_cache_tensors
)
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(27):
if i % 4 == 3 or i == 26:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
assert len(groups) == 4
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
for group in mamba_groups:
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 7 * mla_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
# Each group has one tensor descriptor and all groups alias the same slots.
assert len(kv_cache_config.kv_cache_tensors) == 4
tensors = _tensor_by_layer(kv_cache_config)
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
for index, mla_name in enumerate(mla_names):
mla_offset = _layer_offset(tensors[mla_name], mla_name)
for group in mamba_groups:
if index < len(group.layer_names):
mamba_name = group.layer_names[index]
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_capacity_after_scheduler_unwrap():
"""max_concurrency must survive the scheduler-config unwrap.
Regression for the balanced-mamba hybrid layout:
``generate_scheduler_kv_cache_config`` flattens the MLA
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(45):
if i % 4 == 3:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
# Confirm we are exercising the regression path: the MLA group is no longer
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
assert not any(
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
for g in scheduler_config.kv_cache_groups
)
unwrapped_groups = scheduler_config.kv_cache_groups
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
vllm_config, unwrapped_groups
)
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
expected_blocks_per_request = (
expected_max_mem + expected_pool - 1
) // expected_pool
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
assert max_concurrency > 0
assert max_concurrency == pytest.approx(
scheduler_config.num_blocks / expected_blocks_per_request
)
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
return MLAAttentionSpec(
block_size=block_size,
@@ -3305,9 +3845,10 @@ def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
draft_shares_target_spec to collapse them into one group instead.
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
draft_shares_target_spec to match that dtype while retaining the draft
marker as a group boundary.
"""
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
specs = {
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
@@ -605,3 +606,195 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
}
assert store.puts == expected
def test_worker_lookup_hits_sub_block_partial_tail():
"""worker.lookup must query sub-block keys when partial hash hits are on.
Regression test: ``lookup`` hard-coded ``fine_grained = False``, so the
sub-block keys persisted by ``_sub_block_tail_puts`` were never probed and
partial prefix hits silently returned 0 while the store side kept writing
them. Here the mamba block (16) exceeds the hash unit (4), so
``enable_partial_hash_hits`` is on and the lookup must find the stored
boundary at 12.
"""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
cfg = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=4 * full.page_size_bytes,
layers=["L0"],
layer_stride=4 * full.page_size_bytes,
block_stride=full.page_size_bytes,
),
KVCacheTensor(
size=4 * mamba.page_size_bytes,
layers=["L1"],
layer_stride=4 * mamba.page_size_bytes,
block_stride=mamba.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
],
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
# Hash unit 4 < mamba block 16 -> partial hash hits are enabled.
vllm_config.cache_config.prefix_match_unit = 4
store = _DictStore()
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
worker.tp_size = 1
worker.pp_size = 1
worker.num_kv_head = 8
assert worker.coord.enable_partial_hash_hits
for g_idx, db in enumerate(worker.token_dbs):
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
send_thread = KVCacheStoreSendingThread(
store=store,
token_databases=worker.token_dbs,
block_size=worker.block_size,
coord=worker.coord,
tp_rank=0,
group_put_steps=[1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//4-1]).
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
req = ReqMeta(
req_id="r0",
token_len_chunk=0,
block_ids=([1], [2]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, 7, 12)],
)
send_thread._maybe_offload_boundary_states(req)
worker.store = store
# A 13-token prompt sharing the prefix must hit the stored boundary at 12.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 12
def test_worker_setup_tolerates_finer_scratch_group():
"""Setup and lookup must tolerate a non-prefix-cacheable scratch group.
Regression: GLM-5.3-Flash carries a kpool-tail scratch group whose block
size (``index_kpool`` tokens) is finer than the hash unit, so the
coordinator's divisibility assert and the scratch group's
``ChunkedTokenDatabase`` both rejected worker setup for any hash unit the
scratch block does not divide. Scratch groups never participate in
store/load/lookup, so setup must skip them and lookup must still hit
stored sub-block boundaries on the participating groups.
"""
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
mamba = MambaSpec(
block_size=16,
shapes=((1, 1),),
dtypes=(torch.float32,),
mamba_cache_mode="align",
)
# Scratch block 4 is not divisible by the hash unit 8 below.
scratch = KpoolTailSpec(
block_size=4,
num_kv_heads=2,
head_size=64,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
)
cfg = KVCacheConfig(
num_blocks=4,
kv_cache_tensors=[
KVCacheTensor(
size=4 * full.page_size_bytes,
layers=["L0"],
layer_stride=4 * full.page_size_bytes,
block_stride=full.page_size_bytes,
),
KVCacheTensor(
size=4 * mamba.page_size_bytes,
layers=["L1"],
layer_stride=4 * mamba.page_size_bytes,
block_stride=mamba.page_size_bytes,
),
KVCacheTensor(
size=4 * scratch.page_size_bytes,
layers=["L2"],
layer_stride=4 * scratch.page_size_bytes,
block_stride=scratch.page_size_bytes,
),
],
kv_cache_groups=[
KVCacheGroupSpec(["L0"], full),
KVCacheGroupSpec(["L1"], mamba),
KVCacheGroupSpec(["L2"], scratch),
],
)
vllm_config = _minimal_vllm_config(cache_block_size=16)
# Hash unit 8 divides the participating groups (16) but not the scratch
# group (4); mamba block 16 > 8 keeps partial hash hits on.
vllm_config.cache_config.prefix_match_unit = 8
store = _DictStore()
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
worker.tp_size = 1
worker.pp_size = 1
worker.num_kv_head = 8
assert worker.coord.enable_partial_hash_hits
# The scratch DB is keyed at its own block size and never probed.
assert worker.token_dbs[2].hash_block_size == 4
for g_idx, db in enumerate(worker.token_dbs):
db.set_kv_caches_base_addr([g_idx * 10_000])
db.set_block_len([512])
send_thread = KVCacheStoreSendingThread(
store=store,
token_databases=worker.token_dbs,
block_size=worker.block_size,
coord=worker.coord,
tp_rank=0,
group_put_steps=[1, 1, 1],
kv_role="kv_both",
ready_event=threading.Event(),
replicate_config=MagicMock(),
group_participates=[True, True, False],
)
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
hs = [BlockHash(bytes([i + 1]) * 8) for i in range(3)]
req = ReqMeta(
req_id="r0",
token_len_chunk=0,
block_ids=([1], [2], [3]),
block_hashes=hs,
can_save=True,
num_prompt_tokens=20,
boundary_state_offloads=[(1, 7, 12)],
)
send_thread._maybe_offload_boundary_states(req)
worker.store = store
# A 13-token prompt sharing the prefix must hit the first hash unit.
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
# The scratch group's namespace never enters the store.
scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0]
assert not any(key.startswith(scratch_prefix) for key in store._data)
@@ -211,6 +211,167 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
return worker
@pytest.mark.cpu_test
@pytest.mark.parametrize("logical_block_size", [1152, 640])
@pytest.mark.parametrize("tail_first", [False, True])
def test_register_compressed_indexer_uses_virtual_transfer_pages(
logical_block_size, tail_first
):
"""Compressed indexer rows must split into contiguous NIXL transfer pages."""
from unittest.mock import MagicMock
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
base_worker as bw,
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import (
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
create_kv_cache_views,
)
num_logical_blocks = 3
transfer_block_size = 64
kernel_block_size = 128
tokens_per_state = 4
state_content_bytes = 132
indexer_spec = MLAAttentionSpec(
block_size=logical_block_size,
num_kv_heads=1,
head_size=128,
head_size_v=0,
dtype=torch.uint8,
state_content_bytes=state_content_bytes,
tokens_per_state=tokens_per_state,
)
indexer_page_size = indexer_spec.page_size_bytes
tail_spec = KpoolTailSpec(
block_size=tokens_per_state,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
page_size_padded=indexer_page_size,
sliding_window=tokens_per_state,
)
allocation_size = num_logical_blocks * indexer_page_size
indexer_tensor = KVCacheTensor(
size=allocation_size,
layers=["indexer"],
layer_stride=allocation_size,
block_stride=indexer_page_size,
)
tail_tensor = KVCacheTensor(
size=allocation_size,
layers=["tail"],
layer_stride=allocation_size,
block_stride=indexer_page_size,
)
kv_cache_config = KVCacheConfig(
num_blocks=num_logical_blocks,
kv_cache_tensors=[indexer_tensor, tail_tensor],
kv_cache_groups=[
KVCacheGroupSpec(
["indexer"],
UniformTypeKVCacheSpecs(
block_size=logical_block_size,
kv_cache_specs={"indexer": indexer_spec},
),
),
KVCacheGroupSpec(
["tail"],
UniformTypeKVCacheSpecs(
block_size=tokens_per_state,
kv_cache_specs={"tail": tail_spec},
),
),
],
)
raw = torch.zeros(allocation_size, dtype=torch.int8)
(indexer_cache,) = create_kv_cache_views(
raw,
indexer_spec,
num_logical_blocks,
KVCacheLayout.LBHNC,
indexer_tensor,
kernel_block_size=kernel_block_size,
)
(tail_cache,) = create_kv_cache_views(
raw,
tail_spec,
num_logical_blocks,
KVCacheLayout.LBHNC,
tail_tensor,
)
assert indexer_cache.data_ptr() == tail_cache.data_ptr() == raw.data_ptr()
vllm_config = create_vllm_config(block_size=logical_block_size)
vllm_config.cache_config.kv_cache_layout = "LBHNC"
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
fake_backend = MagicMock()
fake_backend.get_supported_kernel_block_sizes.return_value = [transfer_block_size]
fake_backend.get_name.return_value = "DEEPSEEK_V32_INDEXER"
fake_backend.full_cls_name.return_value = "fake.DEEPSEEK_V32_INDEXER"
fake_platform = MagicMock()
fake_platform.device_type = "cuda"
fake_platform.get_nixl_memory_type.return_value = "VRAM"
caches = [("indexer", indexer_cache), ("tail", tail_cache)]
if tail_first:
caches.reverse()
with (
patch.object(bw, "NixlWrapper", _RecordingNixl),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
patch.object(bw, "current_platform", fake_platform),
set_current_vllm_config(vllm_config),
):
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
worker.use_mla = True
worker.register_kv_caches(dict(caches))
transfer_page_size = transfer_block_size // tokens_per_state * state_content_bytes
num_transfer_blocks = num_logical_blocks * (
logical_block_size // transfer_block_size
)
expected_descs = np.asarray(
[
[
raw.data_ptr() + block_idx * transfer_page_size,
transfer_page_size,
0,
]
for block_idx in range(num_transfer_blocks)
],
dtype=np.uint64,
)
assert worker.block_size == transfer_block_size
assert worker.num_regions == 1
assert worker.block_len_per_layer == [transfer_page_size]
assert worker.block_stride_per_layer == [transfer_page_size]
assert worker._region_is_mla == [True]
assert worker.kv_caches_base_addr[worker.engine_id][0] == [raw.data_ptr()]
assert worker._registered_descs[0] == [(raw.data_ptr(), raw.nbytes, 0, "")]
np.testing.assert_array_equal(worker.src_blocks_data, expected_descs)
assert expected_descs[-1, 0] + expected_descs[-1, 1] == (
raw.data_ptr() + raw.nbytes
)
def _make_remote_meta(
worker,
remote_block_size,
+12 -4
View File
@@ -143,15 +143,22 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
@pytest.mark.parametrize(
("kernel_block_sizes", "expected_num_blocks", "expected_num_states"),
(
"kernel_block_sizes",
"storage_block_size",
"expected_num_blocks",
"expected_num_states",
),
[
(None, 4, 64),
([256], 4, 64),
([64], 16, 16),
(None, None, 4, 64),
([256], None, 4, 64),
([64], None, 16, 16),
([64], 256, 4, 64),
],
)
def test_allocate_compressed_mla_cache(
kernel_block_sizes: list[int] | None,
storage_block_size: int | None,
expected_num_blocks: int,
expected_num_states: int,
):
@@ -161,6 +168,7 @@ def test_allocate_compressed_mla_cache(
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=4,
storage_block_size=storage_block_size,
)
num_pages = 4
config = KVCacheConfig(
@@ -76,6 +76,7 @@ def test_packed_dsv4_zeroer_zeroes_only_each_layers_page():
static_forward_context={
f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(NUM_LAYERS)
},
num_blocks=NUM_BLOCKS,
)
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
@@ -153,6 +154,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
static_forward_context={
name: SimpleNamespace(kv_cache=views[name]) for name in views
},
num_blocks=config.num_blocks,
)
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
+53
View File
@@ -54,6 +54,7 @@ def test_attention_blocks_are_zeroed(spec):
static_forward_context={
layer_name: SimpleNamespace(kv_cache=storage),
},
num_blocks=4,
)
zeroer.zero_block_ids([1])
@@ -64,6 +65,56 @@ def test_attention_blocks_are_zeroed(spec):
assert torch.equal(storage, expected)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_layers_in_one_group_may_use_different_kernel_pages_per_block():
"""Derive kernel pages per block from each layer's allocation."""
device = torch.device("cuda")
num_blocks = 4
spec = SlidingWindowSpec(
block_size=2,
num_kv_heads=1,
head_size=1,
dtype=torch.int32,
sliding_window=2,
)
# Two kernel pages per logical block.
wide = torch.ones((num_blocks * 2, 3), dtype=torch.int32, device=device)
# One kernel page per logical block, carved out of a larger allocation so
# an over-strided write lands in the guard region instead of faulting or
# silently hitting another tensor.
narrow_backing = torch.ones((num_blocks * 3, 5), dtype=torch.int32, device=device)
narrow = narrow_backing[:num_blocks]
zeroer = KVBlockZeroer(
device,
attn_groups_iter=[
AttentionGroup(
None,
["wide", "narrow"],
spec,
0,
)
],
kernel_block_sizes=[1],
static_forward_context={
"wide": SimpleNamespace(kv_cache=wide),
"narrow": SimpleNamespace(kv_cache=narrow),
},
num_blocks=num_blocks,
)
zeroer.zero_block_ids([num_blocks - 1])
torch.accelerator.synchronize()
expected_wide = torch.ones_like(wide)
expected_wide[2 * (num_blocks - 1) :] = 0
assert torch.equal(wide, expected_wide)
expected_narrow = torch.ones_like(narrow_backing)
expected_narrow[num_blocks - 1] = 0
assert torch.equal(narrow_backing, expected_narrow)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
device = torch.device("cuda")
@@ -207,6 +258,7 @@ def test_large_dsv4_launch_geometry(monkeypatch):
name: SimpleNamespace(kv_cache=storage)
for name, storage in storages.items()
},
num_blocks=1,
)
assert zeroer._meta is not None
@@ -332,6 +384,7 @@ def test_zeroes_exactly_one_block_per_layer(layout: KVCacheLayout):
attn_groups_iter=iter(groups),
kernel_block_sizes=[spec.block_size],
static_forward_context=ctx,
num_blocks=num_blocks,
)
zeroer.zero_block_ids([2])
torch.accelerator.synchronize()
@@ -7,15 +7,57 @@ from unittest.mock import Mock
import pytest
import torch
from vllm.config.compilation import CUDAGraphMode
from vllm.platforms import current_platform
from vllm.v1.attention.backends.recoverssm_metadata import (
RecoverSSMMetadata,
RecoverSSMPostprocessMetadata,
)
from vllm.v1.worker.gpu.model_states import mamba_hybrid
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
state = object.__new__(MambaHybridModelState)
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
state.max_model_len = 8192
state._align_mode = False
state.recoverssm = None
positions = torch.tensor([1536], dtype=torch.int64)
input_batch = SimpleNamespace(
num_reqs=1,
num_tokens=1,
num_reqs_after_padding=1,
num_tokens_after_padding=1,
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
seq_lens=torch.tensor([1537], dtype=torch.int32),
is_prefilling_np=torch.tensor([False]).numpy(),
dcp_local_seq_lens=None,
positions=positions,
prompt_lens=torch.tensor([1024], dtype=torch.int32),
)
expected_metadata = {"layer": object()}
build_attn_metadata = Mock(return_value=expected_metadata)
monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata)
metadata = state.prepare_attn(
input_batch=input_batch,
cudagraph_mode=CUDAGraphMode.NONE,
block_tables=(),
slot_mappings=torch.empty(0, dtype=torch.int64),
attn_groups=[],
kv_cache_config=Mock(),
)
assert metadata is expected_metadata
assert build_attn_metadata.call_args.kwargs["positions"] is positions
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
def test_postprocess_state_scalar_with_int32_mapping(