[Model] add GLM-5.3-Flash support (#53906)
This commit is contained in:
@@ -9,13 +9,23 @@ from vllm.platforms import current_platform
|
||||
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE = 128 * 1024 * 1024
|
||||
|
||||
if not current_platform.has_device_capability(100):
|
||||
if not current_platform.is_cuda() or not current_platform.has_device_capability(90):
|
||||
pytest.skip(
|
||||
reason="FlashInfer MLA Requires compute capability of 10 or above.",
|
||||
reason="FlashInfer MLA requires CUDA compute capability 9.0 or above.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
else:
|
||||
from flashinfer.decode import trtllm_batch_decode_with_kv_cache_mla
|
||||
from flashinfer.mla import BatchMLAPagedAttentionWrapper
|
||||
|
||||
requires_sm90 = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(90),
|
||||
reason="This test requires an SM90 GPU.",
|
||||
)
|
||||
requires_sm10x = pytest.mark.skipif(
|
||||
not current_platform.is_device_capability_family(100),
|
||||
reason="This test requires an SM10x GPU.",
|
||||
)
|
||||
|
||||
# Deepseek R1 MLA config.
|
||||
NUM_HEADS = 128
|
||||
@@ -82,6 +92,7 @@ def ref_mla(
|
||||
@pytest.mark.parametrize("dtype", [torch.bfloat16])
|
||||
@pytest.mark.parametrize("bs", [1, 2, 4, 16])
|
||||
@pytest.mark.parametrize("block_size", [32, 64])
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
@@ -119,6 +130,161 @@ def test_flashinfer_mla_decode(dtype: torch.dtype, bs: int, block_size: int):
|
||||
torch.testing.assert_close(out_ans, out_ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_flashinfer_trtllm_sparse_mla_decode_without_rope():
|
||||
"""The native sparse MLA path supports a zero-width rotary tail."""
|
||||
torch.set_default_device("cuda")
|
||||
torch.manual_seed(42)
|
||||
|
||||
batch_size = 2
|
||||
block_size = 64
|
||||
num_blocks = 4
|
||||
sparse_topk = 128
|
||||
valid_lens = torch.tensor([17, 73], dtype=torch.int32)
|
||||
|
||||
query = torch.randn(
|
||||
batch_size,
|
||||
1,
|
||||
NUM_HEADS,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache = torch.randn(
|
||||
num_blocks,
|
||||
block_size,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
|
||||
num_slots = num_blocks * block_size
|
||||
slot_tables = torch.stack(
|
||||
[torch.randperm(num_slots)[:sparse_topk] for _ in range(batch_size)]
|
||||
).to(torch.int32)
|
||||
for row, valid_len in zip(slot_tables, valid_lens.tolist()):
|
||||
row[valid_len:] = -1
|
||||
|
||||
workspace_buffer = torch.empty(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.int8,
|
||||
)
|
||||
out = trtllm_batch_decode_with_kv_cache_mla(
|
||||
query=query,
|
||||
kv_cache=kv_cache.unsqueeze(1),
|
||||
workspace_buffer=workspace_buffer,
|
||||
qk_nope_head_dim=QK_NOPE_HEAD_DIM,
|
||||
kv_lora_rank=KV_LORA_RANK,
|
||||
qk_rope_head_dim=0,
|
||||
block_tables=slot_tables.unsqueeze(1),
|
||||
seq_lens=valid_lens,
|
||||
max_seq_len=sparse_topk,
|
||||
sparse_mla_top_k=sparse_topk,
|
||||
sparse_mla_top_k_lens=valid_lens,
|
||||
bmm1_scale=QK_NOPE_HEAD_DIM**-0.5,
|
||||
bmm2_scale=1.0,
|
||||
).squeeze(1)
|
||||
|
||||
flat_cache = kv_cache.view(num_slots, KV_LORA_RANK).float()
|
||||
refs = []
|
||||
for batch_idx, valid_len in enumerate(valid_lens.tolist()):
|
||||
selected_kv = flat_cache[slot_tables[batch_idx, :valid_len].long()]
|
||||
scores = torch.einsum("hd,kd->hk", query[batch_idx, 0].float(), selected_kv)
|
||||
probs = torch.softmax(scores * QK_NOPE_HEAD_DIM**-0.5, dim=-1)
|
||||
refs.append(torch.einsum("hk,kd->hd", probs, selected_kv))
|
||||
ref = torch.stack(refs).to(torch.bfloat16)
|
||||
|
||||
torch.testing.assert_close(out, ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm90
|
||||
def test_flashinfer_sm90_fp8_mla_decode_without_rope():
|
||||
"""Hopper FA3 supports BF16 queries over an FP8 cache without KPE."""
|
||||
torch.manual_seed(42)
|
||||
device = torch.device("cuda")
|
||||
batch_size = 2
|
||||
num_heads = 16
|
||||
page_size = 16
|
||||
num_pages = 6
|
||||
|
||||
q_nope = torch.randn(
|
||||
batch_size,
|
||||
num_heads,
|
||||
KV_LORA_RANK,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
q_pe = torch.empty(
|
||||
batch_size,
|
||||
num_heads,
|
||||
0,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
ckv = torch.randn(
|
||||
num_pages,
|
||||
page_size,
|
||||
KV_LORA_RANK,
|
||||
device=device,
|
||||
)
|
||||
fp8_max = torch.finfo(torch.float8_e4m3fn).max
|
||||
ckv_scale = ckv.abs().max().item() / fp8_max
|
||||
ckv_fp8 = (ckv / ckv_scale).clamp(-fp8_max, fp8_max).to(torch.float8_e4m3fn)
|
||||
scale_bf16 = torch.tensor(ckv_scale, dtype=torch.bfloat16, device=device)
|
||||
ckv_ref = ckv_fp8.to(torch.bfloat16) * scale_bf16
|
||||
kpe_fp8 = torch.empty(
|
||||
num_pages,
|
||||
page_size,
|
||||
0,
|
||||
dtype=torch.float8_e4m3fn,
|
||||
device=device,
|
||||
)
|
||||
kpe_ref = torch.empty(
|
||||
num_pages,
|
||||
page_size,
|
||||
0,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
)
|
||||
|
||||
qo_indptr = torch.tensor([0, 1, 2], dtype=torch.int32, device=device)
|
||||
kv_indptr = torch.tensor([0, 3, 5], dtype=torch.int32, device=device)
|
||||
kv_indices = torch.tensor([4, 1, 3, 0, 5], dtype=torch.int32, device=device)
|
||||
kv_lens = torch.tensor([45, 29], dtype=torch.int32, device=device)
|
||||
sm_scale = QK_NOPE_HEAD_DIM**-0.5
|
||||
|
||||
def run(
|
||||
ckv_cache: torch.Tensor,
|
||||
kpe_cache: torch.Tensor,
|
||||
**kwargs,
|
||||
) -> torch.Tensor:
|
||||
workspace = torch.empty(
|
||||
FLASHINFER_WORKSPACE_BUFFER_SIZE,
|
||||
dtype=torch.uint8,
|
||||
device=device,
|
||||
)
|
||||
wrapper = BatchMLAPagedAttentionWrapper(workspace, backend="fa3")
|
||||
wrapper.plan(
|
||||
qo_indptr,
|
||||
kv_indptr,
|
||||
kv_indices,
|
||||
kv_lens,
|
||||
num_heads,
|
||||
KV_LORA_RANK,
|
||||
0,
|
||||
page_size,
|
||||
False,
|
||||
sm_scale,
|
||||
q_data_type=torch.bfloat16,
|
||||
kv_data_type=ckv_cache.dtype,
|
||||
)
|
||||
return wrapper.run(q_nope, q_pe, ckv_cache, kpe_cache, **kwargs)
|
||||
|
||||
out_ref = run(ckv_ref, kpe_ref)
|
||||
out = run(ckv_fp8, kpe_fp8, ckv_scale=ckv_scale, kpe_scale=1.0)
|
||||
torch.testing.assert_close(out, out_ref, atol=2e-2, rtol=2e-2)
|
||||
|
||||
|
||||
@requires_sm10x
|
||||
def test_flashinfer_mla_decode_workspace_supports_autotune():
|
||||
"""vLLM's FlashInfer MLA decode workspace must be int8 for autotuning.
|
||||
|
||||
|
||||
@@ -41,6 +41,8 @@ def _make_builder():
|
||||
builder.device = torch.device("cpu")
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=1)
|
||||
builder.model_dtype = torch.bfloat16
|
||||
builder.kv_cache_dtype = "fp8"
|
||||
builder.mla_dims = SimpleNamespace(kv_lora_rank=512, qk_rope_head_dim=64)
|
||||
builder.topk_tokens = topk_tokens
|
||||
builder.req_id_per_token_buffer = torch.zeros(
|
||||
max_num_batched_tokens, dtype=torch.int32, device="cpu"
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
pytest.skip(
|
||||
reason="FlashInfer GDN prefill is not supported on ROCm.",
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import flashinfer.gdn_prefill # noqa: E402
|
||||
|
||||
from vllm.model_executor.layers.mamba.gdn.qwen_gdn_linear_attn import (
|
||||
fi_chunk_gated_delta_rule,
|
||||
) # noqa: E402
|
||||
|
||||
|
||||
def test_flashinfer_gdn_prefill_uses_int64_cu_seqlens(monkeypatch):
|
||||
captured_cu_seqlens = None
|
||||
|
||||
def fake_chunk_gated_delta_rule(**kwargs):
|
||||
nonlocal captured_cu_seqlens
|
||||
captured_cu_seqlens = kwargs["cu_seqlens"]
|
||||
return kwargs["q"]
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer.gdn_prefill,
|
||||
"chunk_gated_delta_rule",
|
||||
fake_chunk_gated_delta_rule,
|
||||
)
|
||||
q = torch.zeros(1, 2, 1, 2)
|
||||
cu_seqlens = torch.tensor([0, 2], dtype=torch.int32)
|
||||
|
||||
output, final_state = fi_chunk_gated_delta_rule(
|
||||
q=q,
|
||||
k=q,
|
||||
v=q,
|
||||
g=torch.zeros(1, 2, 1),
|
||||
beta=torch.zeros(1, 2, 1),
|
||||
initial_state=torch.zeros(1, 1, 2, 2),
|
||||
output_final_state=False,
|
||||
cu_seqlens=cu_seqlens,
|
||||
use_qk_l2norm_in_kernel=False,
|
||||
)
|
||||
|
||||
assert captured_cu_seqlens is not None
|
||||
assert captured_cu_seqlens.dtype == torch.int64
|
||||
assert output.shape == q.shape
|
||||
assert final_state is None
|
||||
@@ -0,0 +1,549 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit test for the batched kpool decode-update kernel.
|
||||
|
||||
Validates ``kpool_decode_update_and_maybe_write_cache_batched`` against an
|
||||
independent pure-torch reference that replicates the per-request, in-position
|
||||
order semantics: stash each token into a paged tail ring; on pool completion
|
||||
(``pos % pool_size == pool_size-1``) softmax(gate+ape)-weighted sum + Hadamard-128
|
||||
+ per-vector fp8 absmax quant + write to the indexer K cache. Covers
|
||||
no-completion, completion-at-end, completion-mid-batch, non-uniform padding,
|
||||
plain decode, plus a randomized fuzz pass.
|
||||
|
||||
The kernel iterates each request's ``next_n`` tokens in position order inside
|
||||
one program (grid = num_requests) to preserve the pool-completion
|
||||
read-after-stash dependency; the reference mirrors that ordering.
|
||||
|
||||
``test_decode_writer_matches_prefill_writer`` is deliberately NOT
|
||||
reference-based: it checks the decode writer against the *prefill* writer
|
||||
(``kpool_compress_and_write_cache``), the invariant that actually matters in
|
||||
production. A hand-written reference can drift to match a buggy kernel -- that
|
||||
is exactly how the stash-gating bug (intra-pool tokens never entering the tail
|
||||
ring, because the stash was gated on the pool-granular ``slot_mapping``) stayed
|
||||
green here.
|
||||
"""
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
kpool_compress_and_write_cache,
|
||||
kpool_decode_update_and_maybe_write_cache_batched,
|
||||
kpool_seed_tail_cache,
|
||||
)
|
||||
else:
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import (
|
||||
kpool_compress_and_write_cache,
|
||||
kpool_decode_update_and_maybe_write_cache_batched,
|
||||
kpool_seed_tail_cache,
|
||||
)
|
||||
|
||||
HEAD_DIM = 128
|
||||
POOL_SIZE = 16
|
||||
PAGE_SIZE = 64
|
||||
NUM_BLOCKS = 32
|
||||
ROUND_SCALE = True
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP8_MAX = torch.finfo(FP8_DTYPE).max
|
||||
|
||||
|
||||
def _make_caches():
|
||||
kv = torch.zeros(
|
||||
NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda"
|
||||
)
|
||||
tail = torch.zeros(
|
||||
NUM_BLOCKS, 2, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda"
|
||||
)
|
||||
return kv, tail
|
||||
|
||||
|
||||
def _tail_slot_for(blocks, pos):
|
||||
"""tail_slot = block*POOL + pos%POOL; each request owns a distinct tail block."""
|
||||
blk = torch.tensor(blocks, device=pos.device, dtype=torch.int32).unsqueeze(1)
|
||||
return (blk * POOL_SIZE + pos % POOL_SIZE).to(torch.int32)
|
||||
|
||||
|
||||
def _seed_prior(tail, blocks, n_prior, seed=42):
|
||||
if n_prior <= 0:
|
||||
return
|
||||
g = torch.Generator(device=tail.device).manual_seed(seed)
|
||||
prior_k = torch.randn(
|
||||
len(blocks),
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=tail.device,
|
||||
generator=g,
|
||||
)
|
||||
prior_s = torch.randn(
|
||||
len(blocks),
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device=tail.device,
|
||||
generator=g,
|
||||
)
|
||||
for i, blk in enumerate(blocks):
|
||||
tail[blk, 0, :n_prior, :] = prior_k[i]
|
||||
tail[blk, 1, :n_prior, :] = prior_s[i]
|
||||
|
||||
|
||||
def _hadamard128_torch(x: torch.Tensor) -> torch.Tensor:
|
||||
"""Reference Hadamard-128 on the last dim (must be 128)."""
|
||||
n = x.shape[-1]
|
||||
assert n == 128
|
||||
h = torch.tensor([[1.0, 1.0], [1.0, -1.0]], dtype=torch.float32, device=x.device)
|
||||
while h.shape[0] < n:
|
||||
h = torch.cat([torch.cat([h, h], dim=1), torch.cat([h, -h], dim=1)], dim=0)
|
||||
h = h / math.sqrt(n)
|
||||
return x @ h
|
||||
|
||||
|
||||
def _torch_reference(
|
||||
kv: torch.Tensor,
|
||||
tail: torch.Tensor,
|
||||
tail_slot: torch.Tensor,
|
||||
key: torch.Tensor,
|
||||
score: torch.Tensor,
|
||||
ape: torch.Tensor,
|
||||
slot_map: torch.Tensor,
|
||||
pos: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Independent reference for the batched decode-update kernel.
|
||||
|
||||
For each request, iterate its next_n tokens in order. On each token:
|
||||
- if pos%POOL == POOL-1 and pos_valid: compress the pool (slots
|
||||
[pool_start..pool_start+POOL-1], current token via is_current) and write
|
||||
fp8 K + fp32 scale to kv_cache at cache_loc.
|
||||
- always stash the current token's K/score into tail[block, pos%POOL].
|
||||
"""
|
||||
kv = kv.clone()
|
||||
tail = tail.clone()
|
||||
B, next_n = pos.shape
|
||||
# The indexer K cache is [num_blocks, PAGE_SIZE, HEAD_DIM+4] uint8 but the
|
||||
# kernels interpret each page as [HEAD_DIM*PAGE_SIZE bytes of K (token-major)
|
||||
# | 4*PAGE_SIZE bytes of fp32 scale (token-major)]. Operate on a flat byte
|
||||
# view so the reference writes K and scale at the exact offsets the kernel
|
||||
# uses (page_base + tok*HEAD_DIM for K; page_base + HEAD_DIM*PAGE_SIZE +
|
||||
# tok*4 for scale).
|
||||
page_bytes = PAGE_SIZE * (HEAD_DIM + 4)
|
||||
k_region = HEAD_DIM * PAGE_SIZE
|
||||
tail_slot_cpu = tail_slot.cpu().tolist()
|
||||
slot_map_cpu = slot_map.cpu().tolist()
|
||||
pos_cpu = pos.cpu().tolist()
|
||||
key_cpu = key.float().cpu()
|
||||
score_cpu = score.float().cpu()
|
||||
ape_cpu = ape.cpu()
|
||||
tail_cpu = tail.float().cpu()
|
||||
kv_flat = kv.view(torch.uint8).reshape(-1).cpu()
|
||||
|
||||
for b in range(B):
|
||||
for t in range(next_n):
|
||||
cache_loc = slot_map_cpu[b][t]
|
||||
p = pos_cpu[b][t]
|
||||
pos_valid = cache_loc >= 0 and p >= 0
|
||||
safe_pos = max(p, 0)
|
||||
slot = safe_pos % POOL_SIZE
|
||||
phys_slot = safe_pos % POOL_SIZE
|
||||
# Per-token block derivation (a leading invalid sentinel must not
|
||||
# poison the base for the rest of the request); clamped like the
|
||||
# kernel so an invalid entry can't form a negative base.
|
||||
block = max(tail_slot_cpu[b][t], 0) // POOL_SIZE
|
||||
|
||||
cur_key = key_cpu[b, t]
|
||||
cur_score = score_cpu[b, t]
|
||||
|
||||
if pos_valid and slot == POOL_SIZE - 1:
|
||||
pool_logical_start = safe_pos - slot
|
||||
pool_scores = []
|
||||
pool_ks = []
|
||||
for ps in range(POOL_SIZE):
|
||||
is_current = ps == slot
|
||||
phys = (pool_logical_start + ps) % POOL_SIZE
|
||||
if is_current:
|
||||
s = cur_score
|
||||
k = cur_key
|
||||
else:
|
||||
s = tail_cpu[block, 1, phys]
|
||||
k = tail_cpu[block, 0, phys]
|
||||
s = s + ape_cpu[ps]
|
||||
pool_scores.append(s)
|
||||
pool_ks.append(k)
|
||||
pool_scores = torch.stack(pool_scores) # [POOL, D]
|
||||
pool_ks = torch.stack(pool_ks) # [POOL, D]
|
||||
max_score = pool_scores.max(dim=0).values
|
||||
prob = torch.exp(pool_scores - max_score)
|
||||
denom = prob.sum(dim=0)
|
||||
acc = (pool_ks * prob).sum(dim=0)
|
||||
x = (acc / denom).to(torch.bfloat16).to(torch.float32)
|
||||
x = _hadamard128_torch(x).to(torch.bfloat16).to(torch.float32)
|
||||
absmax = torch.clamp(x.abs().max(), min=1e-4)
|
||||
if ROUND_SCALE:
|
||||
scale = torch.exp2(torch.ceil(torch.log2(absmax / FP8_MAX)))
|
||||
else:
|
||||
scale = absmax / FP8_MAX
|
||||
quantized = torch.clamp(x / scale, -FP8_MAX, FP8_MAX).to(FP8_DTYPE)
|
||||
# write K and scale at the separated-layout offsets
|
||||
loc = cache_loc
|
||||
loc_page_index = loc // PAGE_SIZE
|
||||
loc_tok = loc % PAGE_SIZE
|
||||
page_base = loc_page_index * page_bytes
|
||||
if current_platform.is_rocm():
|
||||
dims = torch.arange(HEAD_DIM)
|
||||
k_off = (
|
||||
page_base
|
||||
+ (loc_tok // 16) * 16 * HEAD_DIM
|
||||
+ (dims // 16) * 16 * 16
|
||||
+ (loc_tok % 16) * 16
|
||||
+ dims % 16
|
||||
)
|
||||
else:
|
||||
k_off = page_base + loc_tok * HEAD_DIM + torch.arange(HEAD_DIM)
|
||||
s_off = page_base + k_region + loc_tok * 4
|
||||
kv_flat[k_off] = quantized.view(torch.uint8)
|
||||
kv_flat[s_off : s_off + 4] = scale.detach().reshape(1).view(torch.uint8)
|
||||
|
||||
# stash -- gated on the TOKEN-granular tail slot, not on pos_valid.
|
||||
# pos_valid keys off the pool-granular cache_loc, which is -1 for
|
||||
# every token that is not the pool's last, so gating the stash on it
|
||||
# would drop all intra-pool tokens.
|
||||
if p >= 0 and tail_slot_cpu[b][t] >= 0:
|
||||
tail_cpu[block, 0, phys_slot] = cur_key
|
||||
tail_cpu[block, 1, phys_slot] = cur_score
|
||||
|
||||
kv_out = kv_flat.view(NUM_BLOCKS, PAGE_SIZE, HEAD_DIM + 4).to(device="cuda")
|
||||
return kv_out, tail_cpu.to(torch.bfloat16).to(device="cuda")
|
||||
|
||||
|
||||
def _assert_eq(r_ref, r_kern):
|
||||
kv_ref, tail_ref = r_ref
|
||||
kv_kern, tail_kern = r_kern
|
||||
assert torch.equal(kv_ref, kv_kern), (
|
||||
"kv_cache differs: max diff "
|
||||
f"{(kv_ref.int() - kv_kern.int()).abs().max().item()}"
|
||||
)
|
||||
assert torch.equal(tail_ref, tail_kern), (
|
||||
"tail_kv_cache differs: max diff "
|
||||
f"{(tail_ref.float() - tail_kern.float()).abs().max().item()}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_amd_prefill_writer_uses_preshuffled_cache_layout():
|
||||
from vllm.models.glm5next.amd.ops.kpool_compress import (
|
||||
kpool_compress_and_write_cache as amd_kpool_compress,
|
||||
)
|
||||
|
||||
torch.manual_seed(0)
|
||||
token_offset = 17
|
||||
kv = torch.zeros(1, PAGE_SIZE, HEAD_DIM + 4, dtype=torch.uint8, device="cuda")
|
||||
key = torch.randn(1, POOL_SIZE, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
score = torch.randn_like(key)
|
||||
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
|
||||
compressed_k, compressed_scale = amd_kpool_compress(
|
||||
kv,
|
||||
key,
|
||||
score,
|
||||
ape,
|
||||
torch.tensor([token_offset], dtype=torch.int64, device="cuda"),
|
||||
pool_size=POOL_SIZE,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
return_compressed=True,
|
||||
)
|
||||
|
||||
dim = torch.arange(HEAD_DIM, device="cuda")
|
||||
offsets = (
|
||||
(token_offset // 16) * 16 * HEAD_DIM
|
||||
+ (dim // 16) * 16 * 16
|
||||
+ (token_offset % 16) * 16
|
||||
+ dim % 16
|
||||
)
|
||||
flat = kv.view(torch.uint8).reshape(-1)
|
||||
stored_k = flat[offsets].view(compressed_k.dtype)
|
||||
scale_offset = PAGE_SIZE * HEAD_DIM + token_offset * 4
|
||||
stored_scale = flat[scale_offset : scale_offset + 4].view(torch.float32)
|
||||
|
||||
assert torch.equal(stored_k, compressed_k[0])
|
||||
assert torch.equal(stored_scale, compressed_scale)
|
||||
|
||||
|
||||
def _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos):
|
||||
kv = kv.clone()
|
||||
tail = tail.clone()
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv,
|
||||
tail,
|
||||
tail_slot,
|
||||
key,
|
||||
score,
|
||||
ape,
|
||||
slot_map,
|
||||
pos,
|
||||
POOL_SIZE,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
return kv, tail
|
||||
|
||||
|
||||
@pytest.mark.parametrize("pool_size", [4, 16])
|
||||
def test_decode_writer_matches_prefill_writer(pool_size):
|
||||
"""Compare production decode and prefill writers for pool sizes 4 and 16."""
|
||||
n_pools, page, nblk = 8, 64, 4
|
||||
n_tok = n_pools * pool_size
|
||||
dev = "cuda"
|
||||
torch.manual_seed(0)
|
||||
k = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
score = torch.randn(n_tok, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
ape = torch.randn(pool_size, HEAD_DIM, dtype=torch.float32, device=dev)
|
||||
|
||||
kv_prefill = torch.zeros(nblk, page, HEAD_DIM + 4, dtype=torch.uint8, device=dev)
|
||||
kpool_compress_and_write_cache(
|
||||
kv_prefill,
|
||||
k.view(n_pools, pool_size, HEAD_DIM),
|
||||
score.view(n_pools, pool_size, HEAD_DIM),
|
||||
ape,
|
||||
torch.arange(n_pools, dtype=torch.int64, device=dev),
|
||||
pool_size=pool_size,
|
||||
head_dim=HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
# One request owning tail block 0, fed one token per decode step.
|
||||
kv_decode = torch.zeros_like(kv_prefill)
|
||||
tail = torch.zeros(nblk, 2, pool_size, HEAD_DIM, dtype=torch.bfloat16, device=dev)
|
||||
for t in range(n_tok):
|
||||
completes = t % pool_size == pool_size - 1
|
||||
kpool_decode_update_and_maybe_write_cache_batched(
|
||||
kv_decode,
|
||||
tail,
|
||||
# token-granular: every token has a valid tail slot
|
||||
torch.tensor([[t % pool_size]], dtype=torch.int32, device=dev),
|
||||
k[t].view(1, 1, HEAD_DIM),
|
||||
score[t].view(1, 1, HEAD_DIM),
|
||||
ape,
|
||||
# pool-granular: only the pool's last token carries a cache slot
|
||||
torch.tensor(
|
||||
[[t // pool_size if completes else -1]], dtype=torch.int32, device=dev
|
||||
),
|
||||
torch.tensor([[t]], dtype=torch.int32, device=dev),
|
||||
pool_size,
|
||||
HEAD_DIM,
|
||||
round_scale=ROUND_SCALE,
|
||||
)
|
||||
|
||||
differing = [
|
||||
p
|
||||
for p in range(n_pools)
|
||||
if not torch.equal(
|
||||
kv_prefill[p // page, p % page], kv_decode[p // page, p % page]
|
||||
)
|
||||
]
|
||||
assert not differing, (
|
||||
f"decode-written pools differ from prefill-written pools: "
|
||||
f"{len(differing)}/{n_pools} (pool_size={pool_size}, first={differing[:5]})"
|
||||
)
|
||||
|
||||
|
||||
def test_leading_invalid_tail_slot():
|
||||
"""A request whose FIRST token carries an invalid (-1) tail slot while a
|
||||
later token is a real pool completion.
|
||||
|
||||
The tail block must be derived per token, not from token 0: a leading
|
||||
invalid sentinel would otherwise poison the base address for the whole
|
||||
request (out-of-bounds tail reads on the completion).
|
||||
"""
|
||||
torch.manual_seed(0)
|
||||
B, next_n, blocks = 2, 4, [3, 5]
|
||||
# req 0: token 0 invalid (pos -1), tokens 1..3 valid, completion at pos 15
|
||||
# req 1: all valid, no completion
|
||||
pos = torch.tensor(
|
||||
[[-1, 13, 14, 15], [4, 5, 6, 7]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
safe_pos = torch.where(pos >= 0, pos, 0)
|
||||
tail_slot = _tail_slot_for(blocks, safe_pos)
|
||||
# leading invalid entry carries the -1 sentinel, as the scatter path emits
|
||||
tail_slot[0, 0] = -1
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[0, 3] = 15 # req 0 completes its pool on the last verify token
|
||||
|
||||
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
|
||||
|
||||
kv, tail = _make_caches()
|
||||
_seed_prior(tail, blocks, 13)
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
_assert_eq(r_ref, r_kern)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_amd_prefill_seed_honors_padded_tail_block_stride():
|
||||
"""The tail shares a padded indexer allocation in production."""
|
||||
kpool = 4
|
||||
num_blocks = 6
|
||||
logical_block_elems = 2 * kpool * HEAD_DIM
|
||||
padded_block_elems = logical_block_elems + 256
|
||||
sentinel = -123.0
|
||||
backing = torch.full(
|
||||
(num_blocks * padded_block_elems,),
|
||||
sentinel,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
)
|
||||
tail = torch.as_strided(
|
||||
backing,
|
||||
size=(num_blocks, 2, kpool, HEAD_DIM),
|
||||
stride=(padded_block_elems, kpool * HEAD_DIM, HEAD_DIM, 1),
|
||||
)
|
||||
|
||||
block = 3
|
||||
ring_slot = 2
|
||||
key = torch.arange(HEAD_DIM, dtype=torch.bfloat16, device="cuda").unsqueeze(0)
|
||||
score = (key + 256).to(torch.bfloat16)
|
||||
tail_slot = torch.tensor(
|
||||
[block * kpool + ring_slot], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
|
||||
kpool_seed_tail_cache(tail, key, score, tail_slot, kpool, HEAD_DIM)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
assert torch.equal(tail[block, 0, ring_slot], key[0])
|
||||
assert torch.equal(tail[block, 1, ring_slot], score[0])
|
||||
|
||||
compact_offset = (block * 2 * kpool + ring_slot) * HEAD_DIM
|
||||
assert torch.all(backing[compact_offset : compact_offset + HEAD_DIM] == sentinel)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case_id",
|
||||
[
|
||||
"no_completion",
|
||||
"completion_at_end",
|
||||
"completion_mid_batch",
|
||||
"non_uniform_padding",
|
||||
"plain_decode",
|
||||
],
|
||||
)
|
||||
def test_batched_matches_reference(case_id):
|
||||
torch.manual_seed(0)
|
||||
if case_id == "no_completion":
|
||||
B, next_n, blocks = 3, 4, [0, 1, 2]
|
||||
pos = (
|
||||
torch.arange(next_n, device="cuda", dtype=torch.int32)
|
||||
.unsqueeze(0)
|
||||
.expand(B, -1)
|
||||
.contiguous()
|
||||
)
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
n_prior = 0
|
||||
elif case_id == "completion_at_end":
|
||||
B, next_n, blocks = 2, 4, [0, 1]
|
||||
pos = torch.tensor(
|
||||
[[12, 13, 14, 15], [12, 13, 14, 15]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[:, 3] = torch.tensor(
|
||||
[15, PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
n_prior = POOL_SIZE - next_n
|
||||
elif case_id == "completion_mid_batch":
|
||||
B, next_n, blocks = 3, 4, [0, 1, 2]
|
||||
pos = torch.tensor([[13, 14, 15, 16]] * B, dtype=torch.int32, device="cuda")
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[:, 2] = torch.tensor(
|
||||
[15, PAGE_SIZE + 15, 2 * PAGE_SIZE + 15], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
n_prior = 13
|
||||
elif case_id == "non_uniform_padding":
|
||||
B, next_n, blocks = 2, 4, [0, 1]
|
||||
pos = torch.tensor(
|
||||
[[12, 13, 14, 15], [12, 13, -1, -1]], dtype=torch.int32, device="cuda"
|
||||
)
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
slot_map[0, 3] = 15
|
||||
n_prior = POOL_SIZE - 4
|
||||
else: # plain_decode
|
||||
B, next_n, blocks = 4, 1, [0, 1, 2, 3]
|
||||
pos = torch.tensor([[5], [6], [7], [8]], dtype=torch.int32, device="cuda")
|
||||
slot_map = torch.full((B, next_n), -1, dtype=torch.int32, device="cuda")
|
||||
n_prior = 0
|
||||
|
||||
if case_id == "non_uniform_padding":
|
||||
safe_pos = torch.where(pos >= 0, pos, 0)
|
||||
tail_slot = torch.where(pos >= 0, _tail_slot_for(blocks, safe_pos), 0)
|
||||
else:
|
||||
tail_slot = _tail_slot_for(blocks, pos)
|
||||
|
||||
key = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
score = torch.randn(B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda")
|
||||
ape = torch.randn(POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda")
|
||||
|
||||
kv, tail = _make_caches()
|
||||
_seed_prior(tail, blocks, n_prior)
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
_assert_eq(r_ref, r_kern)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("seed", list(range(20)))
|
||||
def test_batched_matches_reference_fuzz(seed):
|
||||
"""Random B / next_n / start positions; covers 0, 1, and multi completion."""
|
||||
g = torch.Generator(device="cuda").manual_seed(seed)
|
||||
B = int(torch.randint(1, 6, (1,), generator=g, device="cuda").item())
|
||||
next_n = int(torch.randint(1, 8, (1,), generator=g, device="cuda").item())
|
||||
blocks = list(range(B))
|
||||
|
||||
starts = torch.randint(0, 33, (B,), generator=g, device="cuda", dtype=torch.int32)
|
||||
pos = starts.unsqueeze(1) + torch.arange(
|
||||
next_n, device="cuda", dtype=torch.int32
|
||||
).unsqueeze(0)
|
||||
tail_slot = _tail_slot_for(blocks, pos)
|
||||
|
||||
is_completion = pos % POOL_SIZE == POOL_SIZE - 1
|
||||
blk = torch.tensor(blocks, device="cuda", dtype=torch.int32).unsqueeze(1)
|
||||
pool_slot = blk * PAGE_SIZE + (POOL_SIZE - 1)
|
||||
slot_map = torch.where(is_completion, pool_slot, torch.full_like(pos, -1))
|
||||
|
||||
key = torch.randn(
|
||||
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
|
||||
)
|
||||
score = torch.randn(
|
||||
B, next_n, HEAD_DIM, dtype=torch.bfloat16, device="cuda", generator=g
|
||||
)
|
||||
ape = torch.randn(
|
||||
POOL_SIZE, HEAD_DIM, dtype=torch.float32, device="cuda", generator=g
|
||||
)
|
||||
|
||||
kv, tail = _make_caches()
|
||||
prior_g = torch.Generator(device="cuda").manual_seed(seed + 1000)
|
||||
for b in range(B):
|
||||
n_prior = int(starts[b].item()) % POOL_SIZE
|
||||
if n_prior > 0:
|
||||
pk = torch.randn(
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
generator=prior_g,
|
||||
)
|
||||
ps = torch.randn(
|
||||
n_prior,
|
||||
HEAD_DIM,
|
||||
dtype=torch.bfloat16,
|
||||
device="cuda",
|
||||
generator=prior_g,
|
||||
)
|
||||
tail[blocks[b], 0, :n_prior, :] = pk
|
||||
tail[blocks[b], 1, :n_prior, :] = ps
|
||||
|
||||
r_ref = _torch_reference(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
r_kern = _run_kernel(kv, tail, tail_slot, key, score, ape, slot_map, pos)
|
||||
_assert_eq(r_ref, r_kern)
|
||||
@@ -5,13 +5,23 @@ from types import SimpleNamespace
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
|
||||
import vllm.model_executor.kernels.mhc # noqa: F401
|
||||
import vllm.model_executor.layers.mhc as mhc_layers
|
||||
from vllm.model_executor.kernels.mhc.tilelang import (
|
||||
_tilelang_hc_prenorm_gemm,
|
||||
_torch_hc_prenorm_gemm,
|
||||
)
|
||||
from vllm.model_executor.layers.mhc import HAS_TILELANG_MHC
|
||||
from vllm.model_executor.layers.mhc import (
|
||||
HAS_AITER_MHC,
|
||||
HAS_AITER_MHC_FUSED,
|
||||
HAS_AITER_MHC_FUSED_NORM,
|
||||
HAS_AITER_MHC_PRE_NORM,
|
||||
HAS_TILELANG_MHC,
|
||||
MHCFusedPostPreOp,
|
||||
MHCPreOp,
|
||||
)
|
||||
from vllm.models.deepseek_v4.nvidia.model import (
|
||||
DeepseekV4DecoderLayer,
|
||||
DeepseekV4Model,
|
||||
@@ -291,6 +301,220 @@ def test_mhc_fused_post_pre(num_tokens, hidden_size, hc_mult):
|
||||
torch.testing.assert_close(x, layer_input_ref, atol=1e-2, rtol=1e-2)
|
||||
|
||||
|
||||
def _rocm_mhc_inputs(num_tokens=2, hidden_size=256, hc_mult=4):
|
||||
residual = torch.randn(
|
||||
(num_tokens, hc_mult, hidden_size), dtype=torch.bfloat16, device=DEVICE
|
||||
)
|
||||
hc_mult3 = 2 * hc_mult + hc_mult * hc_mult
|
||||
fn = (
|
||||
torch.randn(
|
||||
(hc_mult3, hc_mult * hidden_size), dtype=torch.float32, device=DEVICE
|
||||
)
|
||||
* 1e-4
|
||||
)
|
||||
hc_scale = torch.randn((3,), dtype=torch.float32, device=DEVICE) * 0.1
|
||||
hc_base = torch.randn((hc_mult3,), dtype=torch.float32, device=DEVICE) * 0.1
|
||||
norm_weight = torch.randn(hidden_size, dtype=torch.bfloat16, device=DEVICE)
|
||||
return residual, fn, hc_scale, hc_base, norm_weight
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_mhc_pre_rocm_fallback_applies_norm(monkeypatch):
|
||||
set_random_seed(0)
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs()
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
|
||||
sinkhorn_repeat = 20
|
||||
hc_post_alpha = 1.0
|
||||
ref = mhc_pre_ref(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
expected_layer_input = F.rms_norm(
|
||||
ref[2], (ref[2].shape[-1],), norm_weight, norm_eps
|
||||
)
|
||||
monkeypatch.setattr(mhc_layers, "HAS_AITER_MHC", True)
|
||||
monkeypatch.setattr(mhc_layers, "HAS_AITER_MHC_PRE_NORM", False)
|
||||
monkeypatch.setattr(mhc_layers, "HAS_TILELANG_MHC", False)
|
||||
|
||||
out = object.__new__(MHCPreOp).forward_hip(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out[0], ref[0])
|
||||
torch.testing.assert_close(out[1], ref[1])
|
||||
torch.testing.assert_close(out[2], expected_layer_input)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_mhc_fused_rocm_fallback_applies_norm(monkeypatch):
|
||||
set_random_seed(0)
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs()
|
||||
x = torch.randn((2, 256), dtype=torch.bfloat16, device=DEVICE)
|
||||
post_layer_mix = torch.randn((2, 4, 1), dtype=torch.float32, device=DEVICE)
|
||||
comb_res_mix = torch.randn((2, 4, 4), dtype=torch.float32, device=DEVICE)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
|
||||
sinkhorn_repeat = 20
|
||||
hc_post_alpha = 1.0
|
||||
residual_ref = mhc_post_ref(x, residual, post_layer_mix, comb_res_mix)
|
||||
pre_ref = mhc_pre_ref(
|
||||
residual_ref,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
expected_layer_input = F.rms_norm(
|
||||
pre_ref[2], (pre_ref[2].shape[-1],), norm_weight, norm_eps
|
||||
)
|
||||
monkeypatch.setattr(mhc_layers, "HAS_AITER_MHC_FUSED", False)
|
||||
monkeypatch.setattr(mhc_layers, "HAS_TILELANG_MHC", False)
|
||||
|
||||
out = object.__new__(MHCFusedPostPreOp).forward_hip(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out[0], residual_ref)
|
||||
torch.testing.assert_close(out[1], pre_ref[0])
|
||||
torch.testing.assert_close(out[2], pre_ref[1])
|
||||
torch.testing.assert_close(out[3], expected_layer_input)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (current_platform.is_rocm() and HAS_AITER_MHC and HAS_AITER_MHC_PRE_NORM),
|
||||
reason="AITER mHC with fused RMSNorm required",
|
||||
)
|
||||
def test_mhc_pre_rocm_aiter_fuses_norm():
|
||||
set_random_seed(0)
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
|
||||
num_tokens=2, hidden_size=7168
|
||||
)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
|
||||
sinkhorn_repeat = 20
|
||||
hc_post_alpha = 1.0
|
||||
ref = mhc_pre_ref(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
expected_layer_input = F.rms_norm(
|
||||
ref[2], (ref[2].shape[-1],), norm_weight, norm_eps
|
||||
)
|
||||
|
||||
out = object.__new__(MHCPreOp).forward_hip(
|
||||
residual,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out[0], ref[0], atol=5e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(out[1], ref[1], atol=5e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(out[2], expected_layer_input, atol=5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not (
|
||||
current_platform.is_rocm() and HAS_AITER_MHC_FUSED and HAS_AITER_MHC_FUSED_NORM
|
||||
),
|
||||
reason="AITER fused mHC with RMSNorm required",
|
||||
)
|
||||
def test_mhc_fused_rocm_aiter_fuses_norm():
|
||||
set_random_seed(0)
|
||||
residual, fn, hc_scale, hc_base, norm_weight = _rocm_mhc_inputs(
|
||||
num_tokens=2, hidden_size=7168
|
||||
)
|
||||
x = torch.randn((2, 7168), dtype=torch.bfloat16, device=DEVICE)
|
||||
post_layer_mix = torch.randn((2, 4, 1), dtype=torch.float32, device=DEVICE)
|
||||
comb_res_mix = torch.randn((2, 4, 4), dtype=torch.float32, device=DEVICE)
|
||||
rms_eps = hc_pre_eps = hc_sinkhorn_eps = norm_eps = 1e-6
|
||||
sinkhorn_repeat = 20
|
||||
hc_post_alpha = 1.0
|
||||
residual_ref = mhc_post_ref(x, residual, post_layer_mix, comb_res_mix)
|
||||
pre_ref = mhc_pre_ref(
|
||||
residual_ref,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
)
|
||||
expected_layer_input = F.rms_norm(
|
||||
pre_ref[2], (pre_ref[2].shape[-1],), norm_weight, norm_eps
|
||||
)
|
||||
|
||||
out = object.__new__(MHCFusedPostPreOp).forward_hip(
|
||||
x,
|
||||
residual,
|
||||
post_layer_mix,
|
||||
comb_res_mix,
|
||||
fn,
|
||||
hc_scale,
|
||||
hc_base,
|
||||
rms_eps,
|
||||
hc_pre_eps,
|
||||
hc_sinkhorn_eps,
|
||||
hc_post_alpha,
|
||||
sinkhorn_repeat,
|
||||
norm_weight=norm_weight,
|
||||
norm_eps=norm_eps,
|
||||
)
|
||||
|
||||
torch.testing.assert_close(out[0], residual_ref, atol=5e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(out[1], pre_ref[0], atol=5e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(out[2], pre_ref[1], atol=5e-2, rtol=1e-2)
|
||||
torch.testing.assert_close(out[3], expected_layer_input, atol=5e-2, rtol=1e-2)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm required",
|
||||
|
||||
@@ -294,6 +294,7 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
|
||||
"GlmMoeDsaForCausalLM": _HfExamplesInfo(
|
||||
"zai-org/GLM-5", min_transformers_version="5.0.1", is_available_online=False
|
||||
),
|
||||
"Glm5NextForCausalLM": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"GPT2LMHeadModel": _HfExamplesInfo("openai-community/gpt2"),
|
||||
"GPTBigCodeForCausalLM": _HfExamplesInfo(
|
||||
"bigcode/starcoder",
|
||||
@@ -933,6 +934,7 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextForConditionalGeneration": _HfExamplesInfo("zai-org/GLM-5.3-Flash"),
|
||||
"H2OVLChatModel": _HfExamplesInfo(
|
||||
"h2oai/h2ovl-mississippi-800m",
|
||||
trust_remote_code=True,
|
||||
@@ -1703,6 +1705,10 @@ _SPECULATIVE_DECODING_EXAMPLE_MODELS = {
|
||||
speculative_model="zai-org/GLM-OCR",
|
||||
min_transformers_version="5.1.0",
|
||||
),
|
||||
"Glm5NextMTPModel": _HfExamplesInfo(
|
||||
"zai-org/GLM-5.3-Flash",
|
||||
speculative_model="zai-org/GLM-5.3-Flash",
|
||||
),
|
||||
"HYV3MTPModel": _HfExamplesInfo(
|
||||
"tencent/Hy3-preview",
|
||||
speculative_model="tencent/Hy3-preview",
|
||||
|
||||
@@ -142,7 +142,7 @@ def can_initialize(
|
||||
patch.object(V1EngineCore, "_initialize_kv_caches", _initialize_kv_caches_v1),
|
||||
monkeypatch.context() as m,
|
||||
):
|
||||
if requires_spawn_multiprocessing():
|
||||
if requires_spawn_multiprocessing() or model_arch == "Glm5NextForCausalLM":
|
||||
# The EngineCore subprocess re-imports the class and does not
|
||||
# inherit the KV-cache patch above, so it OOMs. Run in-process
|
||||
# so the patch applies.
|
||||
|
||||
@@ -20,6 +20,7 @@ from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
DynamicVideoBackend,
|
||||
Glm5NextVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
Molmo2VideoBackend,
|
||||
Qwen2VLVideoBackend,
|
||||
@@ -1470,3 +1471,233 @@ def test_glm46v_duration_estimation_from_fps():
|
||||
assert len(indices) > 0
|
||||
assert len(indices) % 2 == 0
|
||||
assert all(0 <= idx < 90 for idx in indices)
|
||||
|
||||
|
||||
def test_glm5next_backend_selected_for_processor():
|
||||
"""Glm5NextVideoProcessor maps to the glm5next loader so only the
|
||||
sampled frames are decoded instead of the whole container. Both the
|
||||
borrowed-config spelling and the dedicated Glm5next class name (landing
|
||||
with the new checkpoint) must resolve."""
|
||||
for name in ("Glm5NextVideoProcessor", "Glm5nextVideoProcessor"):
|
||||
assert VIDEO_LOADER_REGISTRY.get_backend_for_video_processor(name) == "glm5next"
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_frames", "original_fps", "duration", "fps", "max_frames"),
|
||||
[
|
||||
(900, 30.0, 30.0, -1, None), # 30s at flat 2.0 raw fps -> 60 frames
|
||||
(3000, 30.0, 100.0, -1, None),
|
||||
(72000, 30.0, 2400.0, -1, None), # 2048 cap
|
||||
(48, 2.0, 24.0, -1, None),
|
||||
(7, 30.0, 10.0, -1, None), # short video -> uniform spread + dedup
|
||||
(300, 25.0, 0, -1, None), # duration derived from frame count
|
||||
(900, 30.0, 30.0, 4, None), # request fps override (raw fps)
|
||||
(900, 30.0, 30.0, -1, 16), # request max_frames override
|
||||
],
|
||||
)
|
||||
def test_glm5next_backend_indices_match_sampler(
|
||||
total_frames, original_fps, duration, fps, max_frames
|
||||
):
|
||||
"""The loader must select exactly the frames the processor's sampler
|
||||
would, with target.fps mapping onto the raw-fps override."""
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
source = VideoSourceMetadata(
|
||||
total_frames_num=total_frames, original_fps=original_fps, duration=duration
|
||||
)
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=fps, max_duration=-1)
|
||||
|
||||
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(
|
||||
source, target, max_frames=max_frames
|
||||
)
|
||||
|
||||
assert indices == glm_sample_frame_indices(
|
||||
total_frames,
|
||||
original_fps,
|
||||
duration,
|
||||
target_fps=fps if fps > 0 else None,
|
||||
max_frame_count=max_frames,
|
||||
)
|
||||
assert len(indices) % 2 == 0
|
||||
assert indices == sorted(indices) # pair padding may repeat the last frame
|
||||
assert all(0 <= idx < total_frames for idx in indices)
|
||||
|
||||
|
||||
def test_glm5next_backend_metadata_contract():
|
||||
"""create_hf_metadata reports the subset so the processor skips
|
||||
re-sampling (do_sample_frames=False) and keeps the original totals."""
|
||||
source = VideoSourceMetadata(total_frames_num=900, original_fps=30.0, duration=30.0)
|
||||
target = VideoTargetMetadata(num_frames=-1, fps=-1, max_duration=-1)
|
||||
indices = Glm5NextVideoBackend.compute_frames_index_to_sample(source, target)
|
||||
|
||||
metadata = Glm5NextVideoBackend.create_hf_metadata(
|
||||
source, indices, video_backend="glm5next"
|
||||
)
|
||||
assert metadata["do_sample_frames"] is False
|
||||
assert metadata["frames_indices"] == indices
|
||||
assert metadata["total_num_frames"] == 900
|
||||
assert metadata["fps"] == 30.0
|
||||
assert metadata["duration"] == 30.0
|
||||
|
||||
# A fully-selected source keeps do_sample_frames=True so the processor's
|
||||
# sampler takes over on the complete frame set.
|
||||
full = list(range(48))
|
||||
assert (
|
||||
Glm5NextVideoBackend.create_hf_metadata(
|
||||
VideoSourceMetadata(total_frames_num=48, original_fps=2.0, duration=24.0),
|
||||
full,
|
||||
video_backend="glm5next",
|
||||
)["do_sample_frames"]
|
||||
is True
|
||||
)
|
||||
|
||||
|
||||
def _write_gray_video(tmp_path, total_frames, fps, size=(32, 32)):
|
||||
"""Synthetic clip whose frame i is flat gray level i (near-lossless under
|
||||
mp4v), so a decoded frame's level maps back to its source index."""
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
|
||||
path = tmp_path / f"gray_{total_frames}_{fps}.mp4"
|
||||
writer = cv2.VideoWriter(
|
||||
str(path), cv2.VideoWriter_fourcc(*"mp4v"), fps, size, isColor=False
|
||||
)
|
||||
assert writer.isOpened()
|
||||
for i in range(total_frames):
|
||||
writer.write(np.full((*size, 1), i, dtype=np.uint8))
|
||||
writer.release()
|
||||
return path
|
||||
|
||||
|
||||
class _CountingCap:
|
||||
"""Proxy over a real capture that counts grab/seek decoding work."""
|
||||
|
||||
def __init__(self, cap):
|
||||
self._cap = cap
|
||||
self.grabs = 0
|
||||
self.seeks = 0
|
||||
|
||||
def get(self, prop):
|
||||
return self._cap.get(prop)
|
||||
|
||||
def grab(self):
|
||||
self.grabs += 1
|
||||
return self._cap.grab()
|
||||
|
||||
def set(self, prop, value):
|
||||
self.seeks += 1
|
||||
return self._cap.set(prop, value)
|
||||
|
||||
def read(self):
|
||||
return self._cap.read()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("backend", ["opencv", "torchcodec"])
|
||||
def test_glm5next_backend_codec_parity(tmp_path, backend):
|
||||
"""Every codec samples the same GLM indices and decodes the same
|
||||
frames; the OpenCV seek reader and torchcodec batched index-exact decode
|
||||
must agree."""
|
||||
if backend == "torchcodec":
|
||||
pytest.importorskip("torchcodec")
|
||||
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
total_frames, fps = 120, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
# Dense default sampling (gap 5) and a sparse max_frames cap (gap 20).
|
||||
for max_frames in (None, 6):
|
||||
kwargs = {} if max_frames is None else {"max_frames": max_frames}
|
||||
expected = glm_sample_frame_indices(
|
||||
total_frames, float(fps), 12.0, max_frame_count=max_frames
|
||||
)
|
||||
|
||||
frames, metadata = Glm5NextVideoBackend.load_bytes(
|
||||
path.read_bytes(), backend=backend, **kwargs
|
||||
)
|
||||
|
||||
assert metadata["frames_indices"] == expected
|
||||
assert metadata["video_backend"].startswith(backend)
|
||||
assert len(frames) == len(expected)
|
||||
for i, idx in enumerate(expected):
|
||||
assert abs(round(float(np.asarray(frames[i]).mean())) - idx) <= 1
|
||||
|
||||
|
||||
def test_glm5next_backend_decodes_only_sampled_frames(tmp_path):
|
||||
"""End to end over a synthetic clip: load_bytes returns exactly the
|
||||
sampler's frame count, with the right frame content at each index."""
|
||||
pytest.importorskip("cv2")
|
||||
|
||||
total_frames, fps = 60, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
glm_sample_frame_indices,
|
||||
)
|
||||
|
||||
expected = glm_sample_frame_indices(total_frames, float(fps), 6.0)
|
||||
|
||||
frames, metadata = Glm5NextVideoBackend.load_bytes(path.read_bytes())
|
||||
|
||||
assert len(frames) == len(expected)
|
||||
assert metadata["frames_indices"] == expected
|
||||
assert metadata["do_sample_frames"] is (len(expected) == total_frames)
|
||||
# Flat gray frames survive mp4v near-losslessly: each decoded frame's
|
||||
# level maps back to its source frame index.
|
||||
for frame, idx in zip(frames, expected):
|
||||
decoded_idx = round(float(np.asarray(frame).mean()))
|
||||
assert abs(decoded_idx - idx) <= 1
|
||||
|
||||
|
||||
def test_glm5next_read_frames_seeks_past_large_gaps(tmp_path):
|
||||
"""Sparse targets must not walk the container: the stock reader grabs
|
||||
every frame up to the last index; the GLM reader seeks instead."""
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
|
||||
total_frames, fps = 200, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
targets = [0, 80, 160, 190] # gaps of 80/80/30 -> only 30 <= threshold
|
||||
|
||||
stock = cv2.VideoCapture(str(path))
|
||||
_, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
|
||||
stock.release()
|
||||
assert stock_indices == targets
|
||||
|
||||
cap = _CountingCap(cv2.VideoCapture(str(path)))
|
||||
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
|
||||
cap._cap.release()
|
||||
|
||||
assert indices == targets == stock_indices
|
||||
for frame, idx in zip(frames, targets):
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
# Walks only the sub-threshold 30-frame hop; the two 80-frame gaps are
|
||||
# seeks. The stock reader grabs all 190 preceding frames.
|
||||
assert cap.grabs <= 29
|
||||
assert cap.seeks == 3
|
||||
|
||||
|
||||
def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
|
||||
"""Dense targets keep the sequential walk (seeking would be slower) and
|
||||
return the same frames as the stock reader."""
|
||||
cv2 = pytest.importorskip("cv2")
|
||||
|
||||
total_frames, fps = 120, 10
|
||||
path = _write_gray_video(tmp_path, total_frames, fps)
|
||||
targets = list(range(0, total_frames, 15)) # gaps of 15 -> all walking
|
||||
|
||||
stock = cv2.VideoCapture(str(path))
|
||||
stock_frames, stock_indices = VideoBackend.read_frames(stock, targets, total_frames)
|
||||
stock.release()
|
||||
|
||||
cap = _CountingCap(cv2.VideoCapture(str(path)))
|
||||
frames, indices = Glm5NextVideoBackend.read_frames(cap, targets, total_frames)
|
||||
cap._cap.release()
|
||||
|
||||
assert indices == stock_indices
|
||||
for frame, stock_frame, idx in zip(frames, stock_frames, targets):
|
||||
assert abs(float(np.asarray(frame).mean()) - float(stock_frame.mean())) <= 2.0
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
# One initial seek, then pure walking -- no re-seek churn.
|
||||
assert cap.seeks == 1
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Unit tests for the vLLM-native GLM-5.3-Flash multimodal processor."""
|
||||
|
||||
import math
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
from PIL import Image
|
||||
from transformers.image_utils import PILImageResampling
|
||||
from transformers.video_utils import VideoMetadata
|
||||
|
||||
from vllm.transformers_utils.processors.glm5next import (
|
||||
Glm5NextImageProcessor,
|
||||
Glm5NextVideoProcessor,
|
||||
_get_pad_content_size,
|
||||
_resize_or_pad,
|
||||
glm_sample_frame_indices,
|
||||
smart_resize,
|
||||
)
|
||||
|
||||
PATCH_SIZE = 14
|
||||
MERGE_SIZE = 2
|
||||
PATCH_EXPAND_FACTOR = 1 # checkpoint processor_config.json
|
||||
FACTOR = PATCH_SIZE * MERGE_SIZE * PATCH_EXPAND_FACTOR # 28
|
||||
PIXELS_PER_TOKEN = 2 * (PATCH_SIZE * MERGE_SIZE) ** 2 # 1568
|
||||
MIN_TOKENS = 16
|
||||
MAX_TOKENS = 800 # serving-cap equivalent budget, keeps test canvases small
|
||||
MIN_PIXELS = MIN_TOKENS * PIXELS_PER_TOKEN
|
||||
MAX_PIXELS = MAX_TOKENS * PIXELS_PER_TOKEN
|
||||
|
||||
|
||||
def resize(height: int, width: int, t: int = 2) -> tuple[int, int]:
|
||||
return smart_resize(
|
||||
t,
|
||||
height,
|
||||
width,
|
||||
t_factor=2,
|
||||
h_factor=FACTOR,
|
||||
w_factor=FACTOR,
|
||||
min_pixels=MIN_PIXELS,
|
||||
max_pixels=MAX_PIXELS,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("height", "width", "t", "expected"),
|
||||
[
|
||||
(300, 400, 2, (308, 420)), # ceil alignment, no budget pressure
|
||||
(50, 80, 2, (112, 168)), # below min budget: proportional upscale
|
||||
(57, 3, 2, (504, 28)), # slim side ceils to one factor
|
||||
(2160, 3840, 2, (588, 1064)), # shrink: max canvas under the cap
|
||||
(112, 11200, 2, (84, 7420)), # extreme aspect stays proportional
|
||||
(300, 400, 16, (252, 308)), # frame count eats into the budget
|
||||
(112, 11200, 16, (28, 2800)), # slim side stays positive, budget holds
|
||||
],
|
||||
)
|
||||
def test_smart_resize_reference_values(height, width, t, expected):
|
||||
got = smart_resize(
|
||||
t,
|
||||
height,
|
||||
width,
|
||||
t_factor=2,
|
||||
h_factor=FACTOR,
|
||||
w_factor=FACTOR,
|
||||
min_pixels=MIN_PIXELS,
|
||||
max_pixels=MAX_PIXELS,
|
||||
)
|
||||
assert got == expected
|
||||
# The aligned canvas never exceeds the budget it was fitted against.
|
||||
t_bar = max(2, round(t / 2) * 2)
|
||||
assert t_bar * got[0] * got[1] <= MAX_PIXELS
|
||||
|
||||
|
||||
def test_smart_resize_stays_snapped_and_positive():
|
||||
heights = (1, 27, 57, 111, 113, 300, 720, 2160)
|
||||
widths = (1, 27, 57, 111, 113, 400, 1280, 11200)
|
||||
for h in heights:
|
||||
for w in widths:
|
||||
for t in (2, 4, 16, 64):
|
||||
resized_h, resized_w = resize(h, w, t)
|
||||
assert resized_h > 0 and resized_w > 0
|
||||
assert resized_h % FACTOR == 0
|
||||
assert resized_w % FACTOR == 0
|
||||
t_bar = max(2, round(t / 2) * 2)
|
||||
assert t_bar * resized_h * resized_w <= MAX_PIXELS
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("height", "width", "match"),
|
||||
[
|
||||
(0, 100, "must be positive"),
|
||||
(100, 0, "must be positive"),
|
||||
],
|
||||
)
|
||||
def test_smart_resize_rejects_degenerate_inputs(height, width, match):
|
||||
with pytest.raises(ValueError, match=match):
|
||||
resize(height, width, 2)
|
||||
|
||||
|
||||
def test_smart_resize_rejects_inverted_budget():
|
||||
with pytest.raises(ValueError, match="min_pixels must be less than or equal"):
|
||||
smart_resize(
|
||||
2,
|
||||
100,
|
||||
100,
|
||||
t_factor=2,
|
||||
h_factor=FACTOR,
|
||||
w_factor=FACTOR,
|
||||
min_pixels=MAX_PIXELS,
|
||||
max_pixels=MIN_PIXELS,
|
||||
)
|
||||
|
||||
|
||||
def test_get_pad_content_size():
|
||||
# Oversized content shrinks proportionally, never upscales by default.
|
||||
assert _get_pad_content_size(300, 400, 308, 420) == (300, 400)
|
||||
assert _get_pad_content_size(600, 800, 308, 420) == (308, 410)
|
||||
# allow_upscale enlarges small content toward the canvas.
|
||||
assert _get_pad_content_size(28, 28, 112, 112, allow_upscale=True) == (112, 112)
|
||||
|
||||
|
||||
def test_resize_or_pad_pads_right_and_bottom():
|
||||
stacked = torch.rand(1, 3, 300, 400)
|
||||
|
||||
def identity_resize(x, size, resample=None):
|
||||
assert (size.height, size.width) == (300, 400)
|
||||
return x
|
||||
|
||||
padded = _resize_or_pad(stacked, 308, 420, "pad", None, identity_resize)
|
||||
assert padded.shape == (1, 3, 308, 420)
|
||||
torch.testing.assert_close(padded[..., :300, :400], stacked)
|
||||
assert padded[..., 300:, :].eq(0).all()
|
||||
assert padded[..., :, 400:].eq(0).all()
|
||||
|
||||
def force_resize(x, size, resample=None):
|
||||
return torch.zeros(x.shape[0], x.shape[1], size.height, size.width)
|
||||
|
||||
assert _resize_or_pad(stacked, 308, 420, "resize", None, force_resize).shape == (
|
||||
1,
|
||||
3,
|
||||
308,
|
||||
420,
|
||||
)
|
||||
with pytest.raises(ValueError, match="resize_mode"):
|
||||
_resize_or_pad(stacked, 308, 420, "crop", None, identity_resize)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def image_processor():
|
||||
return Glm5NextImageProcessor(
|
||||
min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("height", "width"),
|
||||
[(50, 80), (300, 400), (2160, 3840), (112, 11200)],
|
||||
)
|
||||
def test_image_grid_matches_smart_resize(image_processor, height, width):
|
||||
out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
|
||||
resized_h, resized_w = resize(height, width)
|
||||
grid = out["image_grid_thw"][0].tolist()
|
||||
assert grid == [1, resized_h // PATCH_SIZE, resized_w // PATCH_SIZE]
|
||||
assert out["pixel_values"].shape == (
|
||||
grid[1] * grid[2],
|
||||
3 * 2 * PATCH_SIZE * PATCH_SIZE,
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("height", "width"),
|
||||
[(50, 80), (300, 400), (2160, 3840), (112, 11200)],
|
||||
)
|
||||
def test_image_patch_count_matches_preprocess(image_processor, height, width):
|
||||
out = image_processor(Image.new("RGB", (width, height)), return_tensors="pt")
|
||||
grid = out["image_grid_thw"][0].tolist()
|
||||
expected = grid[1] * grid[2]
|
||||
assert image_processor.get_number_of_image_patches(height, width) == expected
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def video_processor():
|
||||
return Glm5NextVideoProcessor(
|
||||
min_image_tokens=MIN_TOKENS, max_image_tokens=MAX_TOKENS
|
||||
)
|
||||
|
||||
|
||||
def run_video_preprocess(video_processor, frames, **overrides):
|
||||
kwargs = dict(
|
||||
resample=PILImageResampling.BICUBIC,
|
||||
image_mean=(0.48145466, 0.4578275, 0.40821073),
|
||||
image_std=(0.26862954, 0.26130258, 0.27577711),
|
||||
patch_size=PATCH_SIZE,
|
||||
temporal_patch_size=2,
|
||||
merge_size=MERGE_SIZE,
|
||||
patch_expand_factor=PATCH_EXPAND_FACTOR,
|
||||
return_tensors="pt",
|
||||
)
|
||||
kwargs.update(overrides)
|
||||
return video_processor._preprocess([frames], **kwargs)
|
||||
|
||||
|
||||
def test_video_preprocess_pads_odd_frame_count(video_processor):
|
||||
out = run_video_preprocess(video_processor, torch.rand(7, 3, 300, 400))
|
||||
grid = out["video_grid_thw"][0].tolist()
|
||||
# 7 frames padded to 8 -> grid_t 4; canvas ceil-aligned (308, 420).
|
||||
assert grid == [4, 22, 30]
|
||||
assert out["pixel_values_videos"].shape == (
|
||||
4 * 22 * 30,
|
||||
3 * 2 * PATCH_SIZE * PATCH_SIZE,
|
||||
)
|
||||
|
||||
|
||||
def test_video_preprocess_keeps_min_side_under_frame_budget(video_processor):
|
||||
out = run_video_preprocess(video_processor, torch.rand(16, 3, 112, 11200))
|
||||
# Budget-fitted canvas (28, 2800): 16 * 28 * 2800 == the pixel cap.
|
||||
assert out["video_grid_thw"][0].tolist() == [8, 2, 200] # height 28, not 0
|
||||
|
||||
|
||||
def test_video_preprocess_pads_content_not_distort(video_processor):
|
||||
frames = torch.zeros(4, 3, 300, 400)
|
||||
frames[..., 50, 60] = 1.0 # marker inside the content area
|
||||
# Bypass rescale/normalize so zero padding stays exactly zero.
|
||||
out = run_video_preprocess(
|
||||
video_processor, frames, do_rescale=False, do_normalize=False
|
||||
)
|
||||
# Pad mode keeps the 300x400 content aspect on the (308, 420) canvas
|
||||
# with zero padding on the right/bottom.
|
||||
grid = out["video_grid_thw"][0].tolist()
|
||||
assert grid == [2, 22, 30]
|
||||
patches = out["pixel_values_videos"].view(
|
||||
grid[0],
|
||||
grid[1] // MERGE_SIZE,
|
||||
grid[2] // MERGE_SIZE,
|
||||
MERGE_SIZE,
|
||||
MERGE_SIZE,
|
||||
3 * 2 * PATCH_SIZE * PATCH_SIZE,
|
||||
)
|
||||
frame = patches[0].permute(0, 3, 1, 4, 2).reshape(grid[1], grid[2], -1)
|
||||
# Patch columns fully right of the 400px content are pure padding.
|
||||
assert frame[:, math.ceil(400 / PATCH_SIZE) :].abs().max() == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("total_frames", "fps", "duration", "expected_len", "first", "last"),
|
||||
[
|
||||
# Dense source: the tp-scaled greedy overshoots extract_t, so the
|
||||
# fixup spreads picks uniformly across the whole video (linspace).
|
||||
(900, 30.0, 30.0, 60, 0, 899),
|
||||
(3000, 30.0, 100.0, 200, 0, 2999),
|
||||
# extract_t capped at 2048 frames.
|
||||
(72000, 30.0, 2400.0, 2048, 0, 71999),
|
||||
# Low container fps: the greedy picks every frame.
|
||||
(48, 2.0, 24.0, 48, 0, 47),
|
||||
# Duration derived from the frame count when metadata lacks it.
|
||||
(300, 25.0, 0, 26, 0, 299),
|
||||
],
|
||||
)
|
||||
def test_glm_sample_frame_indices_behaviour(
|
||||
total_frames, fps, duration, expected_len, first, last
|
||||
):
|
||||
indices = glm_sample_frame_indices(total_frames, fps, duration)
|
||||
assert len(indices) == expected_len
|
||||
assert indices[0] == first
|
||||
assert indices[-1] == last
|
||||
assert len(indices) % 2 == 0
|
||||
assert indices == sorted(indices)
|
||||
assert all(0 <= idx < total_frames for idx in indices)
|
||||
|
||||
|
||||
def test_glm_sample_frame_indices_short_clip_floor_spread():
|
||||
# extract_t (20) > total (7): evenly spaced timestamps, deduplicated,
|
||||
# then pair-padded to an even count.
|
||||
assert glm_sample_frame_indices(7, 30.0, 10.0) == [0, 1, 2, 3, 4, 5, 6, 6]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "expected_len"),
|
||||
[
|
||||
({"target_fps": 0.5}, 16),
|
||||
({"max_frame_count": 16}, 16),
|
||||
({"target_fps": 8}, 240), # 30s * 8 = 240, under the 2048 cap
|
||||
],
|
||||
)
|
||||
def test_glm_sample_frame_indices_request_overrides(kwargs, expected_len):
|
||||
indices = glm_sample_frame_indices(900, 30.0, 30.0, **kwargs)
|
||||
assert len(indices) == expected_len
|
||||
assert len(indices) % 2 == 0
|
||||
|
||||
|
||||
def test_sample_frames_fps_interval(video_processor):
|
||||
indices = video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
|
||||
)
|
||||
assert len(indices) == 60
|
||||
assert indices[0] == 0
|
||||
assert indices[-1] == 899 # uniform spread reaches the final frame
|
||||
|
||||
# Request overrides reach the sampler through both kwarg spellings. The
|
||||
# 15-frame spread is odd, so pair-padding duplicates the last frame.
|
||||
assert (
|
||||
len(
|
||||
video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
|
||||
fps=0.5,
|
||||
)
|
||||
)
|
||||
== 16
|
||||
)
|
||||
assert (
|
||||
len(
|
||||
video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
|
||||
target_fps=0.5,
|
||||
)
|
||||
)
|
||||
== 16
|
||||
)
|
||||
assert (
|
||||
len(
|
||||
video_processor.sample_frames(
|
||||
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
|
||||
max_frames=16,
|
||||
)
|
||||
)
|
||||
== 16
|
||||
)
|
||||
|
||||
|
||||
def test_defaults_mirror_checkpoint_config():
|
||||
"""Bare instantiation matches the checkpoint's ``processor_config.json``
|
||||
token-budget style defaults; ``size`` carries no budget."""
|
||||
image_processor = Glm5NextImageProcessor()
|
||||
assert image_processor.patch_expand_factor == 1
|
||||
assert image_processor.min_image_tokens == 16
|
||||
assert image_processor.max_image_tokens == 8000
|
||||
assert image_processor.resize_mode == "pad"
|
||||
|
||||
video_processor = Glm5NextVideoProcessor()
|
||||
assert video_processor.patch_expand_factor == 1
|
||||
assert video_processor.min_image_tokens == 16
|
||||
assert video_processor.max_image_tokens == 240000
|
||||
assert video_processor.fps_interval == 2.0
|
||||
assert video_processor.max_frame_count_dynamic == 2048
|
||||
|
||||
|
||||
def test_token_budgets_drive_geometry():
|
||||
"""The token bounds fully determine the pixel budget and the grid."""
|
||||
token_proc = Glm5NextImageProcessor(min_image_tokens=64, max_image_tokens=512)
|
||||
img = Image.new("RGB", (400, 300))
|
||||
grid = token_proc(img, return_tensors="pt")["image_grid_thw"][0].tolist()
|
||||
# Budgets 100,352..802,816 px: the 300x400 canvas ceils to (308, 420)
|
||||
# without hitting either bound -> 22x30 patches.
|
||||
assert grid == [1, 22, 30]
|
||||
assert token_proc.get_number_of_image_patches(300, 400) == 22 * 30
|
||||
|
||||
|
||||
def test_missing_token_budgets_rejected():
|
||||
proc = Glm5NextImageProcessor(min_image_tokens=None)
|
||||
with pytest.raises(ValueError, match="min_image_tokens"):
|
||||
proc(Image.new("RGB", (400, 300)), return_tensors="pt")
|
||||
|
||||
|
||||
def test_video_config_fields_land():
|
||||
"""fps_interval / max_frame_count_dynamic from the dedicated config
|
||||
shape sampling without any request overrides."""
|
||||
proc = Glm5NextVideoProcessor(
|
||||
fps_interval=4,
|
||||
max_frame_count_dynamic=32,
|
||||
)
|
||||
indices = proc.sample_frames(
|
||||
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0)
|
||||
)
|
||||
# 30s * 4 = 120 candidates, capped at 32 -> uniform spread of 32.
|
||||
assert len(indices) == 32
|
||||
assert indices[0] == 0
|
||||
assert indices[-1] == 899
|
||||
|
||||
# Request overrides still win over the config values.
|
||||
assert (
|
||||
len(
|
||||
proc.sample_frames(
|
||||
VideoMetadata(total_num_frames=900, fps=30.0, duration=30.0),
|
||||
fps=0.5,
|
||||
)
|
||||
)
|
||||
== 16
|
||||
)
|
||||
@@ -10,6 +10,7 @@ from types import SimpleNamespace
|
||||
from typing import cast
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import pytest
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.model import ModelConfig
|
||||
@@ -19,6 +20,59 @@ from vllm.transformers_utils.config import (
|
||||
get_safetensors_params_metadata,
|
||||
try_get_generation_config,
|
||||
)
|
||||
from vllm.transformers_utils.configs.glm5_next import (
|
||||
Glm5NextConfig,
|
||||
Glm5NextTextConfig,
|
||||
Glm5NextVisionConfig,
|
||||
)
|
||||
|
||||
|
||||
def test_glm5_next_accepts_deepseek_sparse_attention_layers():
|
||||
layer_types = ["linear_attention", "deepseek_sparse_attention"]
|
||||
|
||||
config = Glm5NextTextConfig(
|
||||
num_hidden_layers=len(layer_types), layer_types=layer_types
|
||||
)
|
||||
|
||||
assert config.layer_types == layer_types
|
||||
assert config.layers_block_type == ["linear_attention", "attention"]
|
||||
|
||||
|
||||
def test_glm5_next_accepts_prebuilt_subconfigs():
|
||||
text_config = Glm5NextTextConfig(hidden_size=1024)
|
||||
vision_config = Glm5NextVisionConfig(hidden_size=768)
|
||||
|
||||
config = Glm5NextConfig(
|
||||
text_config=text_config,
|
||||
vision_config=vision_config,
|
||||
)
|
||||
|
||||
assert config.text_config is text_config
|
||||
assert config.vision_config is vision_config
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kwargs", "option"),
|
||||
[
|
||||
(
|
||||
{"index_topk": 2048, "index_dsa_use_layernorm": False},
|
||||
"index_dsa_use_layernorm",
|
||||
),
|
||||
(
|
||||
{"index_topk": 2048, "index_kpool_compress": False},
|
||||
"index_kpool_compress",
|
||||
),
|
||||
(
|
||||
{"index_topk": 2048, "index_kpool_always_select_tail": False},
|
||||
"index_kpool_always_select_tail",
|
||||
),
|
||||
({"hres_vwnstyle": False}, "hres_vwnstyle"),
|
||||
({"mhc_no_norm_weight": True}, "mhc_no_norm_weight"),
|
||||
],
|
||||
)
|
||||
def test_glm5_next_rejects_unimplemented_config_options(kwargs, option):
|
||||
with pytest.raises(NotImplementedError, match=option):
|
||||
Glm5NextTextConfig(**kwargs)
|
||||
|
||||
|
||||
def test_get_llama3_eos_token():
|
||||
|
||||
+7
-6
@@ -22,7 +22,7 @@ from collections.abc import Callable, Iterable, MutableMapping, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from multiprocessing import Process, get_context
|
||||
from pathlib import Path
|
||||
from typing import Any, Literal, cast
|
||||
from typing import TYPE_CHECKING, Any, Literal, cast
|
||||
from unittest.mock import patch
|
||||
|
||||
import anthropic
|
||||
@@ -46,10 +46,6 @@ from vllm.distributed import (
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs
|
||||
from vllm.entrypoints.cli.serve import ServeSubcommand
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.kernels.linear import (
|
||||
_KernelT,
|
||||
init_fp8_linear_kernel,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
QuantKey,
|
||||
)
|
||||
@@ -65,6 +61,9 @@ from vllm.utils.torch_utils import (
|
||||
)
|
||||
from vllm.v1.engine.utils import get_engine_process_shutdown_timeout
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.model_executor.kernels.linear import _KernelT
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
@@ -2349,9 +2348,11 @@ class TestFP8Layer(torch.nn.Module):
|
||||
out_dtype: torch.dtype | None = None,
|
||||
transpose_weights: bool = False,
|
||||
device: torch.device | None = None,
|
||||
force_kernel: type[_KernelT] | None = None,
|
||||
force_kernel: "type[_KernelT] | None" = None,
|
||||
):
|
||||
super().__init__()
|
||||
from vllm.model_executor.kernels.linear import init_fp8_linear_kernel
|
||||
|
||||
self.input_size_per_partition = weight_shape[1]
|
||||
self.output_size_per_partition = weight_shape[0]
|
||||
self.logical_widths = [self.output_size_per_partition]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for error handling in the CUDA attention backend probe.
|
||||
"""Tests for CUDA attention backend selection and probe error handling.
|
||||
|
||||
Environment-shaped probe failures (missing packages, unreadable caches,
|
||||
broken driver installs) must mark the backend unavailable rather than
|
||||
@@ -89,6 +89,48 @@ def test_get_valid_backends_keeps_probing_after_failure():
|
||||
assert valid
|
||||
|
||||
|
||||
def test_sm90_nope_mla_prefers_flashinfer_without_changing_rope_order():
|
||||
backend_cls = MagicMock()
|
||||
backend_cls.validate_configuration.return_value = []
|
||||
sparse_backends = {
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
}
|
||||
|
||||
def sparse_order(head_size: int) -> list[AttentionBackendEnum]:
|
||||
config = SELECTOR_CONFIG._replace(
|
||||
head_size=head_size,
|
||||
use_mla=True,
|
||||
use_sparse=True,
|
||||
)
|
||||
with patch(
|
||||
"vllm.platforms.cuda._get_attn_backend_class",
|
||||
return_value=backend_cls,
|
||||
):
|
||||
valid, _ = CudaPlatform.get_valid_backends(
|
||||
device_capability=SM90,
|
||||
attn_selector_config=config,
|
||||
num_heads=32,
|
||||
)
|
||||
return [
|
||||
candidate.backend
|
||||
for candidate in valid
|
||||
if candidate.backend in sparse_backends
|
||||
]
|
||||
|
||||
assert sparse_order(512) == [
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
]
|
||||
assert sparse_order(576) == [
|
||||
AttentionBackendEnum.FLASH_ATTN_MLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHMLA_SPARSE,
|
||||
AttentionBackendEnum.FLASHINFER_MLA_SPARSE_SM90,
|
||||
]
|
||||
|
||||
|
||||
def test_selected_backend_probe_failure_raises_value_error_with_cause():
|
||||
exc = OSError("libcuda.so.1: cannot open shared object file")
|
||||
with (
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the FlashInfer SM90 sparse MLA backend wiring (no GPU).
|
||||
|
||||
The FlashInfer wrapper and top-k conversion are replaced by CPU recorders;
|
||||
the tests pin the contract between the impl and the kernel API: page_size=1
|
||||
varlen rows, reserved-buffer refresh, plan parameters (dims, NoPE/rope scale,
|
||||
causality), ckv/kpe cache splitting, and the backend's model-shape gates.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
# isort: off
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 as sm90_mod
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse_sm90 import (
|
||||
FlashInferMLASparseSM90Backend,
|
||||
FlashInferMLASparseSM90Builder,
|
||||
FlashInferMLASparseSM90Impl,
|
||||
)
|
||||
# isort: on
|
||||
|
||||
BLOCK_SIZE = 64
|
||||
HEAD = 512
|
||||
TOPK = 128 # triton convert requires width % 128 == 0
|
||||
|
||||
|
||||
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
|
||||
out = torch.full_like(token_indices, -1)
|
||||
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
|
||||
for t in range(token_indices.shape[0]):
|
||||
vals = []
|
||||
for j in range(token_indices.shape[1]):
|
||||
pos = int(token_indices[t, j])
|
||||
if pos == -1:
|
||||
continue
|
||||
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
|
||||
if blk < 0:
|
||||
continue
|
||||
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
|
||||
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
|
||||
counts[t] = len(vals)
|
||||
return out, counts
|
||||
|
||||
|
||||
class FakeWrapper:
|
||||
def __init__(self):
|
||||
self.plan_args = None
|
||||
self.run_args = None
|
||||
|
||||
def plan(self, *args, **kwargs):
|
||||
self.plan_args = (args, kwargs)
|
||||
|
||||
def run(self, *args, **kwargs):
|
||||
q_nope, q_pe, ckv, kpe = args
|
||||
self.run_args = (q_nope, q_pe, ckv, kpe, kwargs)
|
||||
return torch.zeros(
|
||||
q_nope.shape[0], q_nope.shape[1], ckv.shape[-1], dtype=torch.bfloat16
|
||||
)
|
||||
|
||||
|
||||
class FakeState:
|
||||
def __init__(self, width, max_tokens=64):
|
||||
self.kv_indices = torch.zeros(max_tokens * width, dtype=torch.int32)
|
||||
self.kv_len_arr = torch.zeros(max_tokens, dtype=torch.int32)
|
||||
self.wrapper = FakeWrapper()
|
||||
self.plan_calls = []
|
||||
|
||||
def plan(self, num_tokens, kv_lens):
|
||||
self.plan_calls.append((num_tokens, kv_lens))
|
||||
|
||||
|
||||
def make_impl(qk_rope, kv_dtype="fp8_e4m3", num_heads=2, topk_width=TOPK):
|
||||
impl = object.__new__(FlashInferMLASparseSM90Impl)
|
||||
impl.num_heads = num_heads
|
||||
impl.head_size = HEAD + qk_rope
|
||||
impl.scale = (HEAD + qk_rope) ** -0.5
|
||||
impl.kv_lora_rank = HEAD
|
||||
impl.qk_rope_head_dim = qk_rope
|
||||
impl.kv_cache_dtype = kv_dtype
|
||||
impl.use_fp8_kv_cache = kv_dtype in ("fp8", "fp8_e4m3")
|
||||
rows = 4
|
||||
impl.topk_indices_buffer = torch.full((rows, topk_width), -1, dtype=torch.int32)
|
||||
return impl, rows
|
||||
|
||||
|
||||
def make_batch(rows, topk_rows, own_blocks):
|
||||
req_id = torch.tensor([0] * rows, dtype=torch.int32)
|
||||
block_table = torch.zeros(1, 16, dtype=torch.int32)
|
||||
block_table[:, 0] = own_blocks[0]
|
||||
topk = torch.full((rows, TOPK), -1, dtype=torch.int32)
|
||||
for t, row in enumerate(topk_rows):
|
||||
topk[t, : len(row)] = torch.tensor(row, dtype=torch.int32)
|
||||
return SimpleNamespace(
|
||||
req_id_per_token=req_id, block_table=block_table, block_size=BLOCK_SIZE
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("qk_rope,kv_dtype", [(0, "fp8_e4m3"), (64, "auto")])
|
||||
def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
|
||||
impl, rows = make_impl(qk_rope, kv_dtype)
|
||||
state = FakeState(TOPK)
|
||||
monkeypatch.setattr(
|
||||
sm90_mod, "triton_convert_req_index_to_global_index", ref_convert
|
||||
)
|
||||
|
||||
# req with context 10 < topk: 8 valid + -1 padding.
|
||||
topk_rows = [
|
||||
[7, 3, 1, 9, 0, 2, 5, 8] + [-1] * (TOPK - 8),
|
||||
[4, 0, 2, 3, 1] + [-1] * (TOPK - 5),
|
||||
[6, 5, 4] + [-1] * (TOPK - 3),
|
||||
[2] + [-1] * (TOPK - 1),
|
||||
]
|
||||
meta = make_batch(rows, topk_rows, [3])
|
||||
meta.state = state
|
||||
q_nope = torch.randn(rows, impl.num_heads, HEAD)
|
||||
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
|
||||
cache = torch.zeros(
|
||||
8 * BLOCK_SIZE,
|
||||
impl.head_size,
|
||||
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
|
||||
)
|
||||
|
||||
out, lse = impl.forward_mqa(
|
||||
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
|
||||
)
|
||||
assert lse is None and out.shape == (rows, impl.num_heads, HEAD)
|
||||
|
||||
# Reserved buffers carry this step's slots; lengths are NOT refreshed
|
||||
# here (the builder plans them host-side before capture/replay).
|
||||
ref_slots, ref_counts = ref_convert(
|
||||
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
|
||||
)
|
||||
width = TOPK
|
||||
got_slots = state.kv_indices[: rows * width].view(rows, width)
|
||||
for t in range(rows):
|
||||
k = int(ref_counts[t])
|
||||
assert got_slots[t, :k].tolist() == ref_slots[t, :k].tolist()
|
||||
assert state.plan_calls == []
|
||||
|
||||
assert state.wrapper.run_args is not None
|
||||
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
|
||||
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
|
||||
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
|
||||
assert kpe.shape[-1] == qk_rope
|
||||
if impl.use_fp8_kv_cache:
|
||||
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
|
||||
else:
|
||||
assert kwargs == {}
|
||||
|
||||
|
||||
def test_builder_attaches_its_state(monkeypatch):
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
builder.state = FakeState(TOPK)
|
||||
metadata = object.__new__(sm90_mod.FlashInferMLASparseSM90Metadata)
|
||||
metadata.state = None
|
||||
monkeypatch.setattr(
|
||||
sm90_mod.FlashInferMLASparseMetadataBuilder,
|
||||
"build",
|
||||
lambda *_args, **_kwargs: metadata,
|
||||
)
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
query_start_loc_cpu=torch.tensor([0, 1], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1], dtype=torch.int32),
|
||||
positions=None,
|
||||
)
|
||||
|
||||
result = builder.build(0, cam)
|
||||
|
||||
assert result.state is builder.state
|
||||
assert builder.state.plan_calls[0][0] == 1
|
||||
assert builder.state.plan_calls[0][1].tolist() == [1]
|
||||
|
||||
|
||||
def test_plan_uses_state_params(monkeypatch):
|
||||
"""The NoPE/rope dims and scale live on the builder state, not the layer.
|
||||
|
||||
plan() takes exact per-row KV lengths; the schedule is rebuilt on every
|
||||
call (contexts grow between steps) and the indptrs are always full-size
|
||||
with zero-query padding rows past num_tokens.
|
||||
"""
|
||||
impl, rows = make_impl(64, "auto")
|
||||
wrapper = FakeWrapper()
|
||||
state = sm90_mod._SM90State.__new__(sm90_mod._SM90State)
|
||||
state.device = torch.device("cpu")
|
||||
state.wrapper = wrapper
|
||||
state.num_heads = 4
|
||||
state.kv_dtype = torch.bfloat16
|
||||
state.kv_lora_rank = HEAD
|
||||
state.qk_rope_head_dim = 64
|
||||
state.sm_scale = 576**-0.5
|
||||
state.max_tokens = 4
|
||||
state.topk_width = TOPK
|
||||
state.kv_indices = torch.zeros(4 * TOPK)
|
||||
state._arange_cpu = torch.arange(5, dtype=torch.int32)
|
||||
state._qo_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._kv_cpu = torch.empty(5, dtype=torch.int32)
|
||||
state._lens_cpu = torch.full((4,), TOPK, dtype=torch.int32)
|
||||
|
||||
state.plan(3, torch.tensor([2, 5, 7], dtype=torch.int32))
|
||||
assert wrapper.plan_args is not None
|
||||
args, kwargs = wrapper.plan_args
|
||||
(qo, kv, indices, kv_len, heads, ckv, kpe, page, causal, scale) = args
|
||||
assert qo.tolist() == [0, 1, 2, 3, 3] # clamp: rows past 3 have no queries
|
||||
assert kv.tolist() == [i * TOPK for i in (0, 1, 2, 3, 3)]
|
||||
assert kv_len.tolist() == [2, 5, 7, TOPK] # padded row keeps full width
|
||||
assert (heads, ckv, kpe, page, causal) == (4, HEAD, 64, 1, False)
|
||||
assert scale == 576**-0.5
|
||||
assert kwargs["q_data_type"] == torch.bfloat16
|
||||
assert kwargs["kv_data_type"] == torch.bfloat16
|
||||
|
||||
|
||||
def test_kv_lens_host_formula():
|
||||
"""Per-row host lengths: context == position + 1; capped at
|
||||
index_topk + trailing-pool remainder past the sparse threshold."""
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
builder._async_scheduling = False
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=3,
|
||||
query_start_loc_cpu=torch.tensor([0, 5, 7, 10], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([100, 9, 3000], dtype=torch.int32),
|
||||
positions=None,
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 10
|
||||
# req0: positions 95..99 -> ctx 96..100 (all <= 2048: full context)
|
||||
# req1: positions 7,8 -> ctx 8,9
|
||||
# req2: positions 2997..2999 -> ctx 2998..3000 (> 2048: topk + ctx%4)
|
||||
assert lens.tolist() == [96, 97, 98, 99, 100, 8, 9, 2050, 2051, 2048]
|
||||
|
||||
|
||||
def test_kv_lens_host_empty():
|
||||
builder = object.__new__(FlashInferMLASparseSM90Builder)
|
||||
builder._index_topk = 2048
|
||||
builder._index_kpool = 4
|
||||
cam = SimpleNamespace(
|
||||
num_reqs=0,
|
||||
query_start_loc_cpu=torch.tensor([0], dtype=torch.int32),
|
||||
seq_lens=torch.zeros(0, dtype=torch.int32),
|
||||
)
|
||||
num_rows, lens = builder._kv_lens_host(cam)
|
||||
assert num_rows == 0 and lens.numel() == 0
|
||||
|
||||
|
||||
def test_supports_combination_gates(monkeypatch, default_vllm_config):
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: True)
|
||||
call = lambda **kw: FlashInferMLASparseSM90Backend.supports_combination(
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
kv_cache_dtype="fp8_e4m3",
|
||||
block_size=64,
|
||||
use_mla=True,
|
||||
has_sink=False,
|
||||
use_sparse=True,
|
||||
use_mm_prefix=False,
|
||||
device_capability=SimpleNamespace(major=9),
|
||||
**kw,
|
||||
)
|
||||
assert call() is None # no model config: only the feature gate applies
|
||||
|
||||
import vllm.config as cfg
|
||||
|
||||
monkeypatch.setattr(
|
||||
cfg,
|
||||
"get_current_vllm_config",
|
||||
lambda: SimpleNamespace(model_config=None),
|
||||
)
|
||||
assert call() is None
|
||||
monkeypatch.setattr(sm90_mod, "has_flashinfer_sm90_nope_mla", lambda: False)
|
||||
assert "requires FlashInfer" in (call() or "")
|
||||
@@ -0,0 +1,369 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the kpool tail slot mapping (no GPU required).
|
||||
|
||||
The kpool tail cache is a 1-block-per-request circular ring addressed by
|
||||
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
|
||||
allocated per request, never grown, so only column 0 of its block table is
|
||||
ever written; the rest stays zero-initialized).
|
||||
|
||||
The generic per-group slot kernel cannot express that layout: it maps
|
||||
``pos -> bt[req][pos // bs] * bs + pos % bs`` (``_compute_slot_mappings_kernel``
|
||||
in vllm/v1/worker/gpu/block_table.py), so every token at ``pos >= kpool``
|
||||
reads a zero column and collapses onto physical tail block 0. All concurrent
|
||||
requests then share one ``kpool``-slot ring and corrupt each other's pool
|
||||
compression.
|
||||
|
||||
These tests pin that defect's arithmetic, verify the circular replacement
|
||||
(``compute_kpool_tail_slot_mapping``), and mirror the tail kernels' index
|
||||
math to show cross-request pollution before the fix and isolation after.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.v1.attention.backend import CommonAttentionMetadata
|
||||
from vllm.v1.attention.backends.mla.indexer import (
|
||||
KpoolTailBackend,
|
||||
KpoolTailMetadataBuilder,
|
||||
compute_kpool_tail_slot_mapping,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
|
||||
from vllm.v1.kv_cache_layout import KVCacheLayout
|
||||
|
||||
KPOOL = 4
|
||||
|
||||
|
||||
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
|
||||
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
|
||||
spec = KpoolTailSpec(
|
||||
block_size=KPOOL,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=KPOOL,
|
||||
)
|
||||
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
|
||||
_, _, head_stride, state_stride, content_stride = strides
|
||||
|
||||
assert layout is KVCacheLayout.LBHNC
|
||||
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
|
||||
assert state_stride == 128 * torch.bfloat16.itemsize
|
||||
assert content_stride == 1
|
||||
|
||||
|
||||
def make_tail_block_table(own_blocks, width=64):
|
||||
"""Tail-group block table as BlockTables produces it: column 0 holds the
|
||||
request's single KpoolTailManager block, the remaining columns are never
|
||||
written and stay zero."""
|
||||
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
|
||||
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
|
||||
return bt
|
||||
|
||||
|
||||
def legacy_generic_tail_slots(block_table, query_start_loc, positions):
|
||||
"""Reference of the generic ``_compute_slot_mappings_kernel`` arithmetic
|
||||
(block_table.py:305-313) applied to the tail group's table."""
|
||||
slots = []
|
||||
for req in range(block_table.shape[0]):
|
||||
for i in range(query_start_loc[req], query_start_loc[req + 1]):
|
||||
pos = int(positions[i])
|
||||
block_number = int(block_table[req, pos // KPOOL])
|
||||
slots.append(block_number * KPOOL + pos % KPOOL)
|
||||
return torch.tensor(slots, dtype=torch.int64)
|
||||
|
||||
|
||||
def circular_tail_slots(
|
||||
slot_mapping, block_table, query_start_loc, positions, num_actual, num_reqs
|
||||
):
|
||||
return compute_kpool_tail_slot_mapping(
|
||||
slot_mapping,
|
||||
block_table,
|
||||
query_start_loc,
|
||||
positions,
|
||||
num_actual,
|
||||
num_reqs,
|
||||
KPOOL,
|
||||
)
|
||||
|
||||
|
||||
def make_batch(per_req_positions, padded_len=None):
|
||||
positions = torch.cat(
|
||||
[torch.tensor(p, dtype=torch.int64) for p in per_req_positions]
|
||||
)
|
||||
num_actual = positions.numel()
|
||||
num_reqs = len(per_req_positions)
|
||||
lens = [len(p) for p in per_req_positions]
|
||||
qsl = torch.zeros(num_reqs + 1, dtype=torch.int64)
|
||||
torch.cumsum(torch.tensor(lens, dtype=torch.int64), 0, out=qsl[1:])
|
||||
if padded_len is None:
|
||||
padded_len = num_actual
|
||||
slot_mapping = torch.full((padded_len,), -1, dtype=torch.int64)
|
||||
return positions, qsl, slot_mapping, num_actual, num_reqs
|
||||
|
||||
|
||||
def test_legacy_generic_mapping_collapses_onto_block_zero():
|
||||
"""The bug: with the manager's 1-column block table, the generic kernel
|
||||
maps every pos >= kpool onto tail block 0, and distinct requests collide."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))] # prompts of len 10 and 12
|
||||
positions, qsl, _, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
legacy = legacy_generic_tail_slots(bt, qsl, positions)
|
||||
|
||||
# Every token at pos >= kpool resolves to block 0, not the request's own.
|
||||
off = 0
|
||||
for req, prompt in enumerate(per_req):
|
||||
req_slots = legacy[off : off + len(prompt)]
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(req_slots[pos])
|
||||
if pos >= KPOOL:
|
||||
assert slot // KPOOL == 0, (
|
||||
f"expected collapse onto block 0 at pos {pos}"
|
||||
)
|
||||
assert slot // KPOOL != own_blocks[req]
|
||||
off += len(prompt)
|
||||
|
||||
# The two requests share ring slots -> cross-request pollution.
|
||||
a_slots = set(legacy[: len(per_req[0])].tolist())
|
||||
b_slots = set(legacy[len(per_req[0]) :].tolist())
|
||||
assert a_slots & b_slots, "legacy mapping must collide across requests"
|
||||
|
||||
|
||||
def test_circular_mapping_isolates_requests():
|
||||
"""The fix: every token lands in its own request's block at pos % kpool,
|
||||
and no slot is ever shared by two different requests (slots do recur
|
||||
within a request every kpool positions -- that is the circular design)."""
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
|
||||
off = 0
|
||||
per_req_slots = []
|
||||
for req, prompt in enumerate(per_req):
|
||||
req_slots = set()
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(out[off + pos])
|
||||
assert slot // KPOOL == own_blocks[req], (
|
||||
f"req {req} pos {pos} left its tail block"
|
||||
)
|
||||
assert slot % KPOOL == pos % KPOOL
|
||||
req_slots.add(slot)
|
||||
per_req_slots.append(req_slots)
|
||||
off += len(prompt)
|
||||
assert not per_req_slots[0] & per_req_slots[1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prompt_len", [1, 2, 3, 4])
|
||||
def test_circular_mapping_matches_generic_for_short_requests(prompt_len):
|
||||
"""For pos < kpool the generic kernel already picks the own block, so the
|
||||
two mappings agree while every position fits the request's first block
|
||||
(single-request behavior is unchanged)."""
|
||||
own_blocks = [7]
|
||||
per_req = [list(range(prompt_len))]
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(per_req)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
legacy = legacy_generic_tail_slots(bt, qsl, positions)
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
assert torch.equal(out, legacy)
|
||||
|
||||
|
||||
def test_circular_mapping_preserves_padding_and_empty_batch():
|
||||
own_blocks = [5, 9]
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
padded_len = sum(len(p) for p in per_req) + 8
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
|
||||
per_req, padded_len=padded_len
|
||||
)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
|
||||
out = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
|
||||
assert out.shape == slot_mapping.shape
|
||||
assert torch.equal(out[num_actual:], torch.full_like(out[num_actual:], -1))
|
||||
|
||||
empty = circular_tail_slots(slot_mapping, bt, qsl, positions[:0], 0, num_reqs)
|
||||
assert torch.equal(empty, slot_mapping)
|
||||
|
||||
|
||||
def make_common_metadata(per_req_positions, own_blocks, with_positions=True):
|
||||
positions, qsl, slot_mapping, num_actual, num_reqs = make_batch(
|
||||
per_req_positions, padded_len=sum(len(p) for p in per_req_positions) + 4
|
||||
)
|
||||
bt = make_tail_block_table(own_blocks)
|
||||
seq_lens = torch.tensor(
|
||||
[max(p) + 1 if p else 1 for p in per_req_positions], dtype=torch.int64
|
||||
)
|
||||
return CommonAttentionMetadata(
|
||||
query_start_loc=qsl,
|
||||
query_start_loc_cpu=qsl.clone(),
|
||||
seq_lens=seq_lens,
|
||||
num_reqs=num_reqs,
|
||||
num_actual_tokens=num_actual,
|
||||
max_query_len=max((len(p) for p in per_req_positions), default=1),
|
||||
max_seq_len=int(seq_lens.max()) if num_reqs else 1,
|
||||
block_table_tensor=bt,
|
||||
slot_mapping=slot_mapping,
|
||||
positions=positions if with_positions else None,
|
||||
)
|
||||
|
||||
|
||||
def make_tail_builder(block_size=KPOOL, max_num_batched_tokens=128):
|
||||
builder = object.__new__(KpoolTailMetadataBuilder)
|
||||
builder.kv_cache_spec = SimpleNamespace(block_size=block_size)
|
||||
builder.slot_mapping_buffer = torch.empty(max_num_batched_tokens, dtype=torch.int64)
|
||||
return builder
|
||||
|
||||
|
||||
def test_builder_build_uses_circular_mapping():
|
||||
per_req = [list(range(10)), list(range(12))]
|
||||
own_blocks = [5, 9]
|
||||
cam = make_common_metadata(per_req, own_blocks)
|
||||
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
|
||||
|
||||
out = meta.slot_mapping
|
||||
off = 0
|
||||
for req, prompt in enumerate(per_req):
|
||||
for pos in range(len(prompt)):
|
||||
slot = int(out[off + pos])
|
||||
assert slot // KPOOL == own_blocks[req]
|
||||
assert slot % KPOOL == pos % KPOOL
|
||||
off += len(prompt)
|
||||
# Padding tail of the buffer keeps the -1 sentinel.
|
||||
assert torch.equal(
|
||||
out[cam.num_actual_tokens :], torch.full_like(out[cam.num_actual_tokens :], -1)
|
||||
)
|
||||
|
||||
|
||||
def test_builder_build_falls_back_without_positions():
|
||||
"""Capture / dummy builds without positions keep the generic mapping."""
|
||||
per_req = [list(range(10))]
|
||||
cam = make_common_metadata(per_req, [5], with_positions=False)
|
||||
meta = KpoolTailMetadataBuilder.build(make_tail_builder(), 0, cam)
|
||||
assert meta.slot_mapping is cam.slot_mapping
|
||||
|
||||
|
||||
def test_builder_reuses_slot_mapping_storage():
|
||||
builder = make_tail_builder()
|
||||
first = make_common_metadata([list(range(10))], [5])
|
||||
first_meta = KpoolTailMetadataBuilder.build(builder, 0, first)
|
||||
data_ptr = first_meta.slot_mapping.data_ptr()
|
||||
|
||||
second = make_common_metadata([list(range(12))], [9])
|
||||
second_meta = KpoolTailMetadataBuilder.build(builder, 0, second)
|
||||
|
||||
assert second_meta.slot_mapping.data_ptr() == data_ptr
|
||||
assert second_meta.slot_mapping[:12].tolist() == [
|
||||
9 * KPOOL + pos % KPOOL for pos in range(12)
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Index-level mirror of the tail kernels: seed / stash / pool completion
|
||||
# (addressing replicated from kpool_compress.py's Triton kernels).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class TailRingMirror:
|
||||
"""Mirror of _kpool_tail_seed_kernel / _kpool_decode_update_batched_kernel
|
||||
addressing: block = tail_slot // kpool, ring offset = pos % kpool; a pool
|
||||
completing at pos reads ring slots (pool_start + s) % kpool and uses the
|
||||
current token's own K/score for the last member."""
|
||||
|
||||
def __init__(self, num_blocks, kpool=KPOOL):
|
||||
self.kpool = kpool
|
||||
self.k = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
self.s = torch.full((num_blocks, kpool, 3), float("nan"))
|
||||
|
||||
def stash(self, tail_slot, pos, k, s):
|
||||
blk, off = tail_slot // self.kpool, pos % self.kpool
|
||||
self.k[blk, off] = k
|
||||
self.s[blk, off] = s
|
||||
|
||||
seed = stash # the seed kernel writes with the same addressing
|
||||
|
||||
def complete(self, tail_slot, pos, k, s):
|
||||
blk = tail_slot // self.kpool
|
||||
start = pos - (self.kpool - 1)
|
||||
kk = torch.stack(
|
||||
[self.k[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
)
|
||||
ss = torch.stack(
|
||||
[self.s[blk, (start + i) % self.kpool] for i in range(self.kpool)]
|
||||
)
|
||||
kk[-1], ss[-1] = k, s # is_current for the completing token
|
||||
w = torch.softmax(ss, dim=0)
|
||||
return (kk * w).sum(0)
|
||||
|
||||
|
||||
def token_kv(req, pos):
|
||||
k = torch.tensor([pos + 100.0 * req, pos + 0.5, 2.0 * pos + 0.25])
|
||||
s = torch.tensor([0.1 * (pos + 1) + req, 0.2 * pos, 0.05 * pos])
|
||||
return k, s
|
||||
|
||||
|
||||
def tail_slot_for(mapping, req, pos, own_block):
|
||||
if mapping == "legacy":
|
||||
bt_val = own_block if pos < KPOOL else 0
|
||||
return bt_val * KPOOL + pos % KPOOL
|
||||
return own_block * KPOOL + pos % KPOOL
|
||||
|
||||
|
||||
def run_scenario(mapping, interleave):
|
||||
"""Requests A (block 5, prompt len 9) and B (block 9, prompt len 11)
|
||||
decode concurrently; returns A's boundary pool [8, 9, 10, 11]."""
|
||||
ring = TailRingMirror(num_blocks=16)
|
||||
blocks = {"A": 5, "B": 9}
|
||||
prompts = {"A": 9, "B": 11}
|
||||
|
||||
def slot(req, pos):
|
||||
return tail_slot_for(mapping, 0 if req == "A" else 1, pos, blocks[req])
|
||||
|
||||
# Prefill: seed each request's trailing incomplete pool.
|
||||
for req, L in prompts.items():
|
||||
for pos in range(L - (L % KPOOL or KPOOL), L):
|
||||
if pos < 0:
|
||||
continue
|
||||
ring.stash(slot(req, pos), pos, *token_kv(0 if req == "A" else 1, pos))
|
||||
|
||||
# Decode: A emits pos 9, 10, 11; B emits 11, 12, 13. `interleave`
|
||||
# processes B before A within a step, which is exactly what concurrent
|
||||
# Triton programs do when they share tail block 0.
|
||||
order = ["B", "A"] if interleave else ["A", "B"]
|
||||
decode = {"A": [9, 10, 11], "B": [11, 12, 13]}
|
||||
result = None
|
||||
for step in range(3):
|
||||
for req in order:
|
||||
pos = decode[req][step]
|
||||
k, s = token_kv(0 if req == "A" else 1, pos)
|
||||
if pos % KPOOL == KPOOL - 1:
|
||||
pool = ring.complete(slot(req, pos), pos, k, s)
|
||||
if req == "A":
|
||||
result = pool
|
||||
ring.stash(slot(req, pos), pos, k, s)
|
||||
return result
|
||||
|
||||
|
||||
def test_interleaved_decode_pollution_legacy_vs_circular():
|
||||
"""Ground truth: request A decoding alone (its ring used exclusively)."""
|
||||
torch.manual_seed(0)
|
||||
ground_truth = run_scenario("circular", interleave=False)
|
||||
|
||||
legacy = run_scenario("legacy", interleave=True)
|
||||
circular = run_scenario("circular", interleave=True)
|
||||
|
||||
# The old mapping lets request B's tokens into A's ring: A's boundary
|
||||
# pool is compressed from 2 of B's tokens -> wrong.
|
||||
assert not torch.allclose(legacy, ground_truth), (
|
||||
f"legacy mapping unexpectedly clean: {legacy} vs {ground_truth}"
|
||||
)
|
||||
|
||||
# The circular mapping keeps the rings isolated under interleaving.
|
||||
torch.testing.assert_close(circular, ground_truth)
|
||||
@@ -0,0 +1,127 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
|
||||
_use_rocm_sparse_triton,
|
||||
fit_kpool_indices_to_aiter,
|
||||
)
|
||||
from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
|
||||
_sparse_kv_row_offset,
|
||||
_validate_dsv4_sparse_dims,
|
||||
_validate_sparse_dims,
|
||||
)
|
||||
|
||||
|
||||
@triton.jit
|
||||
def _store_sparse_kv_row_offset_kernel(slot_ptr, output_ptr, stride: tl.constexpr):
|
||||
slot = tl.load(slot_ptr)
|
||||
tl.store(output_ptr, _sparse_kv_row_offset(slot, stride))
|
||||
|
||||
|
||||
def test_fit_kpool_indices_preserves_tail_and_best_history():
|
||||
token_indices = torch.tensor(
|
||||
[
|
||||
[10, 9, 8, 7, 6, 5, 100, 101],
|
||||
[10, 9, 8, -1, -1, -1, 100, -1],
|
||||
[-1, -1, -1, -1, -1, -1, -1, -1],
|
||||
],
|
||||
dtype=torch.int32,
|
||||
)
|
||||
|
||||
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=6)
|
||||
|
||||
assert fitted.tolist() == [
|
||||
[10, 9, 8, 7, 100, 101],
|
||||
[10, 9, 8, 100, -1, -1],
|
||||
[-1, -1, -1, -1, -1, -1],
|
||||
]
|
||||
|
||||
|
||||
def test_fit_kpool_indices_exact_width_is_noop():
|
||||
token_indices = torch.tensor([[3, 2, 1, -1]], dtype=torch.int32)
|
||||
|
||||
fitted = fit_kpool_indices_to_aiter(token_indices, topk_tokens=4)
|
||||
|
||||
assert fitted.data_ptr() == token_indices.data_ptr()
|
||||
|
||||
|
||||
def test_fit_kpool_indices_rejects_narrow_input():
|
||||
with pytest.raises(ValueError, match="at least topk_tokens"):
|
||||
fit_kpool_indices_to_aiter(
|
||||
torch.zeros((1, 3), dtype=torch.int32), topk_tokens=4
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
(
|
||||
"kv_cache_dtype",
|
||||
"head_size",
|
||||
"num_prefills",
|
||||
"num_decodes",
|
||||
"num_decode_tokens",
|
||||
"max_query_len",
|
||||
"expected",
|
||||
),
|
||||
[
|
||||
("auto", 512, 1, 0, 0, 32, True),
|
||||
("auto", 512, 1, 2, 2, 32, True),
|
||||
("auto", 512, 0, 2, 2, 1, True),
|
||||
("fp8", 512, 1, 0, 0, 32, False),
|
||||
("auto", 576, 1, 0, 0, 32, False),
|
||||
("auto", 512, 0, 2, 4, 2, False),
|
||||
],
|
||||
)
|
||||
def test_rocm_sparse_triton_route(
|
||||
kv_cache_dtype,
|
||||
head_size,
|
||||
num_prefills,
|
||||
num_decodes,
|
||||
num_decode_tokens,
|
||||
max_query_len,
|
||||
expected,
|
||||
):
|
||||
assert (
|
||||
_use_rocm_sparse_triton(
|
||||
kv_cache_dtype=kv_cache_dtype,
|
||||
head_size=head_size,
|
||||
kv_lora_rank=512,
|
||||
num_prefills=num_prefills,
|
||||
num_decodes=num_decodes,
|
||||
num_decode_tokens=num_decode_tokens,
|
||||
max_query_len=max_query_len,
|
||||
)
|
||||
is expected
|
||||
)
|
||||
|
||||
|
||||
def test_rocm_sparse_attention_accepts_glm_nope_dimensions():
|
||||
_validate_sparse_dims(512, 512, 0, "test")
|
||||
|
||||
|
||||
def test_rocm_sparse_attention_rejects_inconsistent_dimensions():
|
||||
with pytest.raises(AssertionError, match="expected head_dim"):
|
||||
_validate_sparse_dims(511, 512, 0, "test")
|
||||
|
||||
|
||||
def test_dsv4_sparse_attention_keeps_layout_constraint():
|
||||
_validate_dsv4_sparse_dims(512, 448, 64, "test")
|
||||
with pytest.raises(AssertionError, match="expects 448 NoPE dims"):
|
||||
_validate_dsv4_sparse_dims(512, 512, 0, "test")
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_rocm(), reason="ROCm required")
|
||||
def test_sparse_prefill_kv_row_offset_does_not_overflow_int32():
|
||||
# GLM's 640-token pages cross the signed-int32 address boundary at block
|
||||
# 6554 for a 512-element KV row. The production kernel must promote the
|
||||
# slot before multiplying by the row stride.
|
||||
slot = torch.tensor([6554 * 640], dtype=torch.int32, device="cuda")
|
||||
output = torch.empty(1, dtype=torch.int64, device="cuda")
|
||||
|
||||
_store_sparse_kv_row_offset_kernel[(1,)](slot, output, stride=512)
|
||||
|
||||
assert output.item() == 6554 * 640 * 512
|
||||
@@ -0,0 +1,184 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""CPU tests for the decode top-k row->token mapping (no GPU required).
|
||||
|
||||
On non-uniform decode batches (``requires_padding`` -- mixed plain-decode and
|
||||
spec-verify requests, or variable MTP verify lens; taken on Hopper where the
|
||||
varlen/flatten logits path is unavailable), the pool-topk rows follow the
|
||||
PADDED ``[batch_size, next_n]`` grid: row ``(b, t)`` is flat decode token
|
||||
``offset_b + t``. The former inline ``dec_seq = positions[:n] + 1``
|
||||
(``n = batch_size * next_n``) indexes the flat per-token layout with padded
|
||||
coordinates, so rows after the first non-uniform request read another
|
||||
request's positions and rows past the decode region read prefill tokens --
|
||||
``expand_pools_and_append_tail`` then anchors the tail at a foreign length.
|
||||
|
||||
These tests pin the defect with the exact production arithmetic and verify
|
||||
the layout-aware replacement (``_decode_topk_seq_lens``), including the tail
|
||||
expansion consequences via the pure-torch expand/append pair that the fused
|
||||
kernel is documented to replicate.
|
||||
"""
|
||||
|
||||
import torch
|
||||
|
||||
# Bootstrap the glm5next package before entering the indexer module: its
|
||||
# kpool_compress import runs glm5next/__init__, which pulls model ->
|
||||
# attention -> back into sparse_attn_indexer_kpool (attention.py imports the
|
||||
# class at module scope). Production always enters via attention.py first.
|
||||
# isort: off
|
||||
import vllm.models.glm5next # noqa: F401
|
||||
from vllm.models.glm5next.nvidia.ops.kpool_compress import ( # noqa: E402
|
||||
append_tail_to_topk,
|
||||
expand_pools_to_tokens,
|
||||
)
|
||||
# isort: on
|
||||
|
||||
import vllm.model_executor.layers.sparse_attn_indexer_kpool as indexer_mod
|
||||
from vllm.model_executor.layers.sparse_attn_indexer_kpool import (
|
||||
_decode_topk_seq_lens,
|
||||
_fill_short_decode_causal_indices,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
KPOOL = 4
|
||||
TOPK_TOKENS = 16
|
||||
SELECT_K = TOPK_TOKENS // KPOOL
|
||||
|
||||
|
||||
def test_kpool_ops_dispatch_matches_platform():
|
||||
expected_backend = ".amd." if current_platform.is_rocm() else ".nvidia."
|
||||
assert expected_backend in indexer_mod.kpool_ops.__name__
|
||||
|
||||
|
||||
def test_short_decode_fills_exact_causal_rows():
|
||||
topk = torch.full((3, 8), 99, dtype=torch.int32)
|
||||
positions = torch.tensor([0, 3, 7], dtype=torch.int64)
|
||||
|
||||
assert _fill_short_decode_causal_indices(topk, positions, 3, 8, 8)
|
||||
assert topk.tolist() == [
|
||||
[0, -1, -1, -1, -1, -1, -1, -1],
|
||||
[0, 1, 2, 3, -1, -1, -1, -1],
|
||||
[0, 1, 2, 3, 4, 5, 6, 7],
|
||||
]
|
||||
|
||||
|
||||
def test_short_decode_leaves_buffer_unchanged_for_sparse_context():
|
||||
topk = torch.full((2, 8), 99, dtype=torch.int32)
|
||||
before = topk.clone()
|
||||
|
||||
assert not _fill_short_decode_causal_indices(topk, torch.tensor([7, 8]), 2, 9, 8)
|
||||
assert torch.equal(topk, before)
|
||||
|
||||
|
||||
def make_non_uniform_batch():
|
||||
"""3 requests: plain decode (1 token), MTP verify (4), adaptive verify (3).
|
||||
|
||||
Flat decode positions (production layout: decode tokens first, then any
|
||||
prefill tokens of the same batch):
|
||||
req0: [30] (context len 30)
|
||||
req1: [100..103] (verify at context len 100)
|
||||
req2: [7, 8, 9] (verify at context len 7)
|
||||
Followed by 4 prefill tokens at positions 555..558 so the flat tensor is
|
||||
at least ``n = 3 * 4`` long, as it is in a real mixed batch.
|
||||
"""
|
||||
decode_lens = torch.tensor([1, 4, 3], dtype=torch.int64)
|
||||
per_req_positions = [[30], [100, 101, 102, 103], [7, 8, 9]]
|
||||
flat_decode = [p for req in per_req_positions for p in req]
|
||||
positions = torch.tensor(flat_decode + [555, 556, 557, 558], dtype=torch.int64)
|
||||
return decode_lens, per_req_positions, positions
|
||||
|
||||
|
||||
def expected_row_seq_lens(per_req_positions, batch_size, next_n):
|
||||
"""Ground truth: row (b, t) -> pos + 1 for real rows, 0 for pad rows."""
|
||||
out = torch.zeros(batch_size * next_n, dtype=torch.int32)
|
||||
for b, req in enumerate(per_req_positions):
|
||||
for t, pos in enumerate(req):
|
||||
out[b * next_n + t] = pos + 1
|
||||
return out
|
||||
|
||||
|
||||
def test_legacy_flat_layout_misaligns_and_bleeds():
|
||||
"""The bug: ``positions[:n] + 1`` with padded-row coordinates reads other
|
||||
requests' positions and prefill positions."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
batch_size = decode_lens.shape[0]
|
||||
next_n = int(decode_lens.max())
|
||||
n = batch_size * next_n
|
||||
assert n == 12
|
||||
|
||||
legacy = positions[:n].to(torch.int32) + 1
|
||||
expected = expected_row_seq_lens(per_req, batch_size, next_n)
|
||||
|
||||
# Row (1, 0): req1's first verify token (pos 100, seq 101) reads flat
|
||||
# index 4 -> req1's LAST verify token (pos 103). Its true tail token is
|
||||
# dropped and the tail anchors 3 tokens late.
|
||||
assert int(legacy[4]) == 104 and int(expected[4]) == 101
|
||||
|
||||
# Rows (2, *): req2 starts at flat offset 5, but padded coordinates point
|
||||
# at flat indices 8..11 -- the batch's PREFILL positions.
|
||||
for t in range(3):
|
||||
assert int(legacy[2 * next_n + t]) >= 556, (
|
||||
"legacy row (2, t) should read prefill positions"
|
||||
)
|
||||
assert not torch.equal(legacy, expected)
|
||||
|
||||
|
||||
def test_helper_uniform_layout_matches_flat_slice():
|
||||
"""Uniform batches keep the flat shortcut (zero behavior/perf change)."""
|
||||
per_req = [[200, 201, 202, 203], [50, 51, 52, 53]]
|
||||
positions = torch.tensor([p for r in per_req for p in r], dtype=torch.int64)
|
||||
decode_lens = torch.tensor([4, 4], dtype=torch.int64)
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 2, 4, requires_padding=False)
|
||||
assert torch.equal(out, positions[:8].to(torch.int32) + 1)
|
||||
|
||||
|
||||
def test_helper_padded_layout_per_row():
|
||||
"""Non-uniform batches map every padded row to its own token's position;
|
||||
pad rows get 0 (empty tail)."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
out = _decode_topk_seq_lens(positions, decode_lens, 8, 3, 4, requires_padding=True)
|
||||
expected = expected_row_seq_lens(per_req, 3, 4)
|
||||
assert torch.equal(out, expected)
|
||||
# Pad rows (0, 1..3) and (2, 3) collapse to 0 -> no tail appended.
|
||||
assert out[1] == 0 and out[2] == 0 and out[3] == 0 and out[11] == 0
|
||||
|
||||
|
||||
def expand_tail_region(dec_seq):
|
||||
"""Run the production pure-torch expand + append pair (the fused kernel
|
||||
replicates it exactly on the identity path) and return the tail columns
|
||||
[TOPK_TOKENS, TOPK_TOKENS + KPOOL - 1)."""
|
||||
rows = dec_seq.shape[0]
|
||||
pool_ids = torch.arange(SELECT_K, dtype=torch.int64).expand(rows, SELECT_K)
|
||||
valid = torch.ones_like(pool_ids, dtype=torch.bool)
|
||||
expanded = expand_pools_to_tokens(pool_ids, valid, TOPK_TOKENS, KPOOL)
|
||||
seq_lens = dec_seq.to(torch.int32)
|
||||
pool_lens = (seq_lens // KPOOL).to(torch.int32)
|
||||
out = append_tail_to_topk(expanded, seq_lens, pool_lens, KPOOL)
|
||||
return out[:, TOPK_TOKENS:]
|
||||
|
||||
|
||||
def test_tail_expansion_legacy_vs_fixed():
|
||||
"""End-to-end tail consequence: the fixed mapping appends exactly the
|
||||
request's trailing incomplete pool; the legacy flat slice drops real tail
|
||||
tokens and emits indices far past the request's own sequence."""
|
||||
decode_lens, per_req, positions = make_non_uniform_batch()
|
||||
n = 12
|
||||
fixed = _decode_topk_seq_lens(
|
||||
positions, decode_lens, 8, 3, 4, requires_padding=True
|
||||
)
|
||||
legacy = positions[:n].to(torch.int32) + 1
|
||||
|
||||
fixed_tail = expand_tail_region(fixed)
|
||||
legacy_tail = expand_tail_region(legacy)
|
||||
|
||||
# Fixed: row (1, 0) (seq 101) keeps its single tail token 100; row (2, 2)
|
||||
# (seq 10) keeps its full trailing pool [8, 9].
|
||||
assert fixed_tail[4, 0].item() == 100
|
||||
assert fixed_tail[10, :2].tolist() == [8, 9]
|
||||
|
||||
# Legacy: row (1, 0) loses its tail token (anchored at 104, count 0)...
|
||||
assert legacy_tail[4, 0].item() == -1
|
||||
# ...and row (2, 2) reads prefill position 557 -> tail indices 556/557,
|
||||
# way past req2's 10-token sequence -> out-of-bounds block-table reads.
|
||||
assert legacy_tail[10, :2].tolist() == [556, 557]
|
||||
|
||||
assert not torch.equal(legacy_tail, fixed_tail)
|
||||
@@ -40,11 +40,13 @@ if not current_platform.is_cuda():
|
||||
allow_module_level=True,
|
||||
)
|
||||
|
||||
import vllm.v1.attention.backends.mla.flashinfer_mla_sparse as flashinfer_sparse_mod
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
from vllm.utils.math_utils import cdiv
|
||||
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
|
||||
FlashInferMLASparseImpl,
|
||||
FlashInferMLASparseTRTLLMBackend,
|
||||
)
|
||||
from vllm.v1.attention.backends.mla.flashmla_sparse import (
|
||||
@@ -82,6 +84,60 @@ SPARSE_BACKEND_BATCH_SPECS["large_q_pure_prefill"] = BatchSpec(
|
||||
DEVICE_TYPE = current_platform.device_type
|
||||
|
||||
|
||||
def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
|
||||
"""Weight absorption must not change the model's attention temperature."""
|
||||
model_scale = 256**-0.5
|
||||
kv_lora_rank = 512
|
||||
topk = torch.zeros((1, 1), dtype=torch.int32)
|
||||
metadata = SimpleNamespace(
|
||||
req_id_per_token=torch.zeros(1, dtype=torch.int32),
|
||||
block_table=torch.zeros((1, 1), dtype=torch.int32),
|
||||
block_size=1,
|
||||
)
|
||||
recorded_scale = None
|
||||
|
||||
impl = object.__new__(FlashInferMLASparseImpl)
|
||||
impl.scale = model_scale
|
||||
impl.qk_nope_head_dim = 256
|
||||
impl.kv_lora_rank = kv_lora_rank
|
||||
impl.qk_rope_head_dim = 0
|
||||
impl.kv_cache_dtype = "auto"
|
||||
impl.topk_indices_buffer = topk
|
||||
impl.dcp_world_size = 1
|
||||
impl._workspace_buffer = torch.empty(1)
|
||||
impl.bmm1_scale = None
|
||||
impl.bmm2_scale = None
|
||||
impl.is_nope_mla = True
|
||||
impl.need_to_return_lse_for_decode = False
|
||||
monkeypatch.setattr(
|
||||
flashinfer_sparse_mod,
|
||||
"triton_convert_req_index_to_global_index",
|
||||
lambda *args, **kwargs: (topk, torch.ones(1, dtype=torch.int32)),
|
||||
)
|
||||
|
||||
import flashinfer.decode
|
||||
|
||||
def fake_flashinfer(**kwargs):
|
||||
nonlocal recorded_scale
|
||||
recorded_scale = kwargs["bmm1_scale"]
|
||||
return torch.zeros((1, 1, 1, kv_lora_rank))
|
||||
|
||||
monkeypatch.setattr(
|
||||
flashinfer.decode,
|
||||
"trtllm_batch_decode_with_kv_cache_mla",
|
||||
fake_flashinfer,
|
||||
)
|
||||
impl.forward_mqa(
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
torch.zeros(1, 1, kv_lora_rank),
|
||||
metadata,
|
||||
SimpleNamespace(),
|
||||
)
|
||||
|
||||
assert recorded_scale == model_scale
|
||||
assert recorded_scale != kv_lora_rank**-0.5
|
||||
|
||||
|
||||
def _float_to_e8m0_truncate(f: float) -> float:
|
||||
"""Simulate SM100's float -> e8m0 -> bf16 scale conversion.
|
||||
e8m0 format only stores the exponent (power of 2).
|
||||
@@ -1451,14 +1507,14 @@ def test_triton_convert_returns_valid_counts(num_topk_tokens: int):
|
||||
return_valid_counts=False,
|
||||
)
|
||||
assert isinstance(result_only, torch.Tensor)
|
||||
torch.testing.assert_close(result_only, result, rtol=0, atol=0)
|
||||
for row, num_valid in enumerate(expected_valid):
|
||||
compact_valid = result[row, :num_valid].sort().values
|
||||
original_valid = result_only[row][result_only[row] >= 0].sort().values
|
||||
torch.testing.assert_close(compact_valid, original_valid, rtol=0, atol=0)
|
||||
assert torch.all(result[row, num_valid:] == -1)
|
||||
|
||||
|
||||
def test_flashmla_cache_dtype_aliases_use_ds_layout():
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
_canonicalize_sparse_mla_kv_cache_dtype,
|
||||
)
|
||||
|
||||
# kv-cache dtype aliases are canonicalized to fp8_ds_mla before the layer
|
||||
# stores kv_cache_dtype, so they cannot bypass the gate.
|
||||
for alias in ("fp8", "fp8_e4m3"):
|
||||
|
||||
@@ -23,6 +23,7 @@ from vllm.v1.core.kv_cache_utils import (
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
@@ -1801,6 +1802,127 @@ def test_hybrid_sliding_window_group_disables_partial_hash_hits():
|
||||
assert len(computed_blocks.blocks[0]) * hash_block_size == num_computed
|
||||
|
||||
|
||||
def test_opted_out_scratch_group_keeps_partial_hash_hits():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
block_size=mamba_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=mamba_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
|
||||
|
||||
def test_kpool_tail_supports_128_token_partial_hash_hits():
|
||||
hash_block_size = 128
|
||||
cache_block_size = 9 * hash_block_size
|
||||
kpool_block_size = 64
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=cache_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=cache_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
KpoolTailSpec(
|
||||
block_size=kpool_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=kpool_block_size,
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
assert manager.coordinator.scheduler_block_size == cache_block_size
|
||||
assert manager.coordinator.hash_block_size == hash_block_size
|
||||
assert manager.coordinator.enable_partial_hash_hits
|
||||
|
||||
shared_prefix = [10] * (12 * hash_block_size)
|
||||
req0 = make_request("0", shared_prefix, hash_block_size, sha256)
|
||||
computed_blocks, num_computed, _ = manager.get_computed_blocks(req0)
|
||||
assert (
|
||||
manager.allocate_slots(req0, len(shared_prefix), num_computed, computed_blocks)
|
||||
is not None
|
||||
)
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("1", shared_prefix + [12] * 64, hash_block_size, sha256)
|
||||
_, num_computed, _ = manager.get_computed_blocks(req1)
|
||||
assert num_computed == len(shared_prefix)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dcp_world_size", [1, 2, 4])
|
||||
def test_hybrid_partial_hash_hit_uses_cow_under_dcp(dcp_world_size: int):
|
||||
hash_block_size = 2
|
||||
|
||||
@@ -6,13 +6,19 @@ import importlib
|
||||
from collections.abc import Callable
|
||||
from dataclasses import replace
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from typing import Any, cast
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
import vllm.v1.core.kv_cache_utils as kv_cache_utils
|
||||
from vllm.config import CacheConfig, ModelConfig, SchedulerConfig, VllmConfig
|
||||
from vllm.config import (
|
||||
CacheConfig,
|
||||
KVTransferConfig,
|
||||
ModelConfig,
|
||||
SchedulerConfig,
|
||||
VllmConfig,
|
||||
)
|
||||
from vllm.config.kv_events import KVEventsConfig
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.inputs import (
|
||||
@@ -47,6 +53,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
FullAttentionSpec,
|
||||
HiddenStateCacheSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheSpec,
|
||||
@@ -2191,34 +2198,567 @@ def test_generate_scheduler_kv_cache_config():
|
||||
)
|
||||
|
||||
|
||||
def test_mixed_precision_kv_cache_with_uniform_type_specs():
|
||||
fp8_spec = new_kv_cache_spec(dtype=torch.float8_e4m3fn)
|
||||
bf16_spec = new_kv_cache_spec(dtype=torch.bfloat16)
|
||||
worker_config = KVCacheConfig(
|
||||
num_blocks=10,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["fp8_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16, kv_cache_specs={"fp8_layer": fp8_spec}
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["bf16_layer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=16, kv_cache_specs={"bf16_layer": bf16_spec}
|
||||
),
|
||||
),
|
||||
],
|
||||
def _glm5_like_kv_cache_spec(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> tuple[dict[str, KVCacheSpec], list[str]]:
|
||||
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
mamba_layers = []
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
name = f"layers.{i}.linear_attn"
|
||||
kv_cache_spec[name] = mamba_spec_factory()
|
||||
mamba_layers.append(name)
|
||||
return kv_cache_spec, mamba_layers
|
||||
|
||||
|
||||
def _glm5_like_kv_cache_spec_with_tail(
|
||||
mamba_spec_factory=new_mamba_spec,
|
||||
) -> dict[str, KVCacheSpec]:
|
||||
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
|
||||
|
||||
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
|
||||
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
|
||||
is why the fixture drops the tokens_per_state=16 shape used above.
|
||||
"""
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
|
||||
for i in range(3, 45, 4):
|
||||
kpool = 4
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = replace(
|
||||
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
|
||||
tokens_per_state=kpool,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
|
||||
block_size=kpool,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=kpool,
|
||||
)
|
||||
return kv_cache_spec
|
||||
|
||||
|
||||
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
|
||||
return {
|
||||
layer_name: tensor
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
for layer_name in tensor.layers
|
||||
}
|
||||
|
||||
|
||||
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
|
||||
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
|
||||
|
||||
|
||||
def test_get_kv_cache_config_balanced_mamba_hybrid():
|
||||
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
uniform_groups = [
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
]
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
|
||||
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
|
||||
# gets an MLA slot.
|
||||
assert len(groups) == 5
|
||||
assert len(uniform_groups) == 1
|
||||
assert len(uniform_groups[0].layer_names) == 22
|
||||
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 4].layer_names
|
||||
# Mamba pages are padded up to the MLA page.
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_padded == mla_page
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
# Every block id is charged the full per-block sum.
|
||||
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
scheduler_config = generate_scheduler_kv_cache_config([worker_config])
|
||||
|
||||
assert worker_config.needs_kv_cache_zeroing
|
||||
assert scheduler_config.needs_kv_cache_zeroing
|
||||
available_memory = bytes_per_block * 100 + 1
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, available_memory
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
|
||||
# Every logical layer has a view into one backing allocation. Mamba views
|
||||
# alias their corresponding MLA slot by using the same byte offset.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
|
||||
for i, mla_name in enumerate(mla_layer_names):
|
||||
mla_tensor = tensors[mla_name]
|
||||
assert mla_tensor.block_stride == mla_page
|
||||
for group in mamba_groups:
|
||||
if i < len(group.layer_names):
|
||||
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
|
||||
for i in range(11):
|
||||
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
|
||||
|
||||
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
|
||||
assert total_allocated == bytes_per_block * 100
|
||||
assert 0 <= available_memory - total_allocated < bytes_per_block
|
||||
|
||||
assert get_max_concurrency_for_kv_cache_config(
|
||||
vllm_config, kv_cache_config
|
||||
) == pytest.approx(100 / blocks_per_request)
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size=16):
|
||||
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
|
||||
"""The kpool tail parasitizes the indexer tensors instead of getting its
|
||||
own: sibling idx/tail tensors paired by layer order, zero standalone tail
|
||||
bytes, one shared block per request, and no prefix-caching leakage from
|
||||
the kpool-sized scratch group."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
|
||||
assert tail_logical_page < idx_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
tail_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and all(
|
||||
isinstance(spec, KpoolTailSpec)
|
||||
for spec in group.kv_cache_spec.kv_cache_specs.values()
|
||||
)
|
||||
)
|
||||
# The tail never prefix-caches; its page is padded up to the indexer page
|
||||
# so the runner's strided view rides the indexer storage.
|
||||
assert not tail_group.kv_cache_spec.prefix_cacheable
|
||||
tail_inner = cast(
|
||||
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
|
||||
)
|
||||
assert tail_inner.page_size_padded == idx_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Tail layers get logical views but no additional storage: each view aliases
|
||||
# its sibling indexer at the same offset in the shared allocation.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 67
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
for i in range(11):
|
||||
idx_name = f"layers.{4 * i + 3}.indexer"
|
||||
tail_name = f"layers.{4 * i + 3}.tail"
|
||||
assert tensors[idx_name].offset == tensors[tail_name].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
# The layout detector surfaces the sibling names in layer order for the
|
||||
# accounting and connector paths.
|
||||
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
|
||||
assert layout is not None
|
||||
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
|
||||
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
|
||||
|
||||
# Accounting charges exactly one extra shared block per request for the
|
||||
# tail, not a per-sequence allocation.
|
||||
attn_group = next(
|
||||
group
|
||||
for group in groups
|
||||
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
and group is not tail_group
|
||||
)
|
||||
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
|
||||
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
|
||||
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
|
||||
assert (
|
||||
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
|
||||
== blocks_per_request * bytes_per_block
|
||||
)
|
||||
|
||||
|
||||
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
|
||||
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
|
||||
prefix-cache hash granularity: participating groups alone decide it."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def align_mamba():
|
||||
return new_mamba_spec(mamba_cache_mode="align")
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(
|
||||
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=1,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=groups,
|
||||
)
|
||||
hash_vllm_config = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(
|
||||
block_size=16,
|
||||
enable_prefix_caching=True,
|
||||
prefix_match_unit=None,
|
||||
),
|
||||
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
|
||||
kv_transfer_config=object(),
|
||||
)
|
||||
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
|
||||
# the lcm including the tail (1024 % 4 == 0, so it coincides).
|
||||
assert kv_cache_utils.resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, hash_vllm_config
|
||||
) == (1024, 16)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
|
||||
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
|
||||
def big_mamba_spec():
|
||||
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert big_mamba_spec().page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(ValueError, match="does not fit the MLA page"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
|
||||
"""Use the generic layout error when no kpool indexer is present."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
|
||||
shapes=((512, 1024),), dtypes=(torch.float32,)
|
||||
)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
|
||||
|
||||
with pytest.raises(NotImplementedError, match="page size"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
|
||||
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
|
||||
disable slot sharing; the layout re-pads them to the MLA page."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
def prepadded_mamba_spec():
|
||||
return new_mamba_spec(page_size_padded=294_912)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
assert prepadded_mamba_spec().page_size_bytes < mla_page
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert len(groups) == 5
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
|
||||
"""Round-robin mamba grouping keeps practical PP splits balanced: every
|
||||
stage's largest projected mamba group slice fits its projected MLA
|
||||
layers, so per-stage slot tensors all have an MLA owner."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
|
||||
# PP=2 split at transformer layer 22/23.
|
||||
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
|
||||
# layers, projected mamba groups [5, 5, 4, 4].
|
||||
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
|
||||
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
global_groups, worker_spec
|
||||
)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 5 * mla_page + 5 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
# Every projected Mamba slot aliases an MLA owner.
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
|
||||
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
|
||||
# The last slot only hosts the two groups with a 5th projected layer.
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
|
||||
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
|
||||
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
|
||||
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
|
||||
grouping bumps to 5 groups so every stage's projection keeps sharing
|
||||
on instead of silently falling back on stage 0."""
|
||||
from vllm.config import ParallelConfig
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=4),
|
||||
)
|
||||
|
||||
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
|
||||
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
|
||||
for k, name in enumerate(mamba_layers):
|
||||
assert name in mamba_groups[k % 5].layer_names
|
||||
|
||||
for rank in range(4):
|
||||
start, end = get_pp_indices(45, rank, 4)
|
||||
worker_spec = {
|
||||
n: s
|
||||
for n, s in kv_cache_spec.items()
|
||||
if start <= int(n.split(".")[1]) < end
|
||||
}
|
||||
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
|
||||
groups, worker_spec
|
||||
)
|
||||
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
|
||||
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
|
||||
from vllm.config import ParallelConfig
|
||||
|
||||
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
|
||||
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
|
||||
vllm_config = VllmConfig(
|
||||
model_config=ModelConfig(max_model_len=8192),
|
||||
parallel_config=ParallelConfig(pipeline_parallel_size=2),
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
|
||||
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
|
||||
"""Hybrid slot sharing must take precedence over the experimental
|
||||
enable_cross_layers_blocks packed layout: generic packing would give the
|
||||
MLA slots a strided view, breaking the contiguous virtual split."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
kv_transfer_config = KVTransferConfig(
|
||||
kv_connector="NixlConnector",
|
||||
kv_role="kv_both",
|
||||
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
|
||||
)
|
||||
vllm_config = VllmConfig(
|
||||
model_config=model_config, kv_transfer_config=kv_transfer_config
|
||||
)
|
||||
|
||||
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 11 * mla_page + 11 * idx_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 56
|
||||
assert all(
|
||||
tensor.block_stride in (mla_page, idx_page)
|
||||
for tensor in kv_cache_config.kv_cache_tensors
|
||||
)
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
|
||||
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBNHC"
|
||||
|
||||
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(27):
|
||||
if i % 4 == 3 or i == 26:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
mamba_groups = [
|
||||
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
|
||||
]
|
||||
assert len(groups) == 4
|
||||
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
|
||||
for group in mamba_groups:
|
||||
assert group.kv_cache_spec.page_size_bytes == mla_page
|
||||
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
assert bytes_per_block == 7 * mla_page
|
||||
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
assert kv_cache_config.num_blocks == 100
|
||||
# Each group has one tensor descriptor and all groups alias the same slots.
|
||||
assert len(kv_cache_config.kv_cache_tensors) == 4
|
||||
tensors = _tensor_by_layer(kv_cache_config)
|
||||
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
|
||||
for index, mla_name in enumerate(mla_names):
|
||||
mla_offset = _layer_offset(tensors[mla_name], mla_name)
|
||||
for group in mamba_groups:
|
||||
if index < len(group.layer_names):
|
||||
mamba_name = group.layer_names[index]
|
||||
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
|
||||
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
|
||||
|
||||
|
||||
def test_get_kv_cache_capacity_after_scheduler_unwrap():
|
||||
"""max_concurrency must survive the scheduler-config unwrap.
|
||||
|
||||
Regression for the balanced-mamba hybrid layout:
|
||||
``generate_scheduler_kv_cache_config`` flattens the MLA
|
||||
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
|
||||
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
|
||||
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
|
||||
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
|
||||
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
|
||||
"""
|
||||
model_config = ModelConfig(max_model_len=8192)
|
||||
vllm_config = VllmConfig(model_config=model_config)
|
||||
|
||||
kv_cache_spec: dict[str, KVCacheSpec] = {}
|
||||
for i in range(45):
|
||||
if i % 4 == 3:
|
||||
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=576,
|
||||
dtype=torch.bfloat16,
|
||||
)
|
||||
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
|
||||
block_size=1024,
|
||||
num_kv_heads=1,
|
||||
head_size=132,
|
||||
dtype=torch.uint8,
|
||||
tokens_per_state=16,
|
||||
)
|
||||
else:
|
||||
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
|
||||
|
||||
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
|
||||
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
|
||||
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
|
||||
vllm_config, groups, bytes_per_block * 100 + 1
|
||||
)
|
||||
|
||||
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
|
||||
# Confirm we are exercising the regression path: the MLA group is no longer
|
||||
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
|
||||
assert not any(
|
||||
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
|
||||
for g in scheduler_config.kv_cache_groups
|
||||
)
|
||||
|
||||
unwrapped_groups = scheduler_config.kv_cache_groups
|
||||
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
|
||||
vllm_config, unwrapped_groups
|
||||
)
|
||||
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
|
||||
expected_blocks_per_request = (
|
||||
expected_max_mem + expected_pool - 1
|
||||
) // expected_pool
|
||||
|
||||
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
|
||||
assert max_concurrency > 0
|
||||
assert max_concurrency == pytest.approx(
|
||||
scheduler_config.num_blocks / expected_blocks_per_request
|
||||
)
|
||||
|
||||
|
||||
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
|
||||
# head_size = kv_lora_rank(512) + qk_rope_head_dim(64) = 576
|
||||
return MLAAttentionSpec(
|
||||
block_size=block_size,
|
||||
@@ -3305,9 +3845,10 @@ def _hybrid_specs_with_draft(draft: bool, draft_shares_target_spec: bool = False
|
||||
"""A K3-shaped hybrid: MLA full attention + Mamba, optionally plus a
|
||||
DSpark-style draft MLA layer marked non_causal_multi_token_decode.
|
||||
|
||||
The target's fp8 KV dtype is what keeps the draft in its own bucket, as it
|
||||
does on Kimi-K3 (target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to collapse them into one group instead.
|
||||
The target's fp8 KV dtype is one difference from the draft on Kimi-K3
|
||||
(target `--kv-cache-dtype fp8_e4m3`, draft `auto`). Pass
|
||||
draft_shares_target_spec to match that dtype while retaining the draft
|
||||
marker as a group boundary.
|
||||
"""
|
||||
target_dtype = None if draft_shares_target_spec else "fp8_e4m3"
|
||||
specs = {
|
||||
|
||||
@@ -30,6 +30,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
from vllm.v1.core.kv_cache_utils import BlockHash
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
@@ -605,3 +606,195 @@ def test_sub_block_partial_tail_offload_covers_smaller_group_blocks():
|
||||
token_dbs[1].key_for(hs[2]): [10_000 + mamba_cow_block * 512],
|
||||
}
|
||||
assert store.puts == expected
|
||||
|
||||
|
||||
def test_worker_lookup_hits_sub_block_partial_tail():
|
||||
"""worker.lookup must query sub-block keys when partial hash hits are on.
|
||||
|
||||
Regression test: ``lookup`` hard-coded ``fine_grained = False``, so the
|
||||
sub-block keys persisted by ``_sub_block_tail_puts`` were never probed and
|
||||
partial prefix hits silently returned 0 while the store side kept writing
|
||||
them. Here the mamba block (16) exceeds the hash unit (4), so
|
||||
``enable_partial_hash_hits`` is on and the lookup must find the stored
|
||||
boundary at 12.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=4 * full.page_size_bytes,
|
||||
layers=["L0"],
|
||||
layer_stride=4 * full.page_size_bytes,
|
||||
block_stride=full.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * mamba.page_size_bytes,
|
||||
layers=["L1"],
|
||||
layer_stride=4 * mamba.page_size_bytes,
|
||||
block_stride=mamba.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
# Hash unit 4 < mamba block 16 -> partial hash hits are enabled.
|
||||
vllm_config.cache_config.prefix_match_unit = 4
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.coord.enable_partial_hash_hits
|
||||
|
||||
for g_idx, db in enumerate(worker.token_dbs):
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//4-1]).
|
||||
hs = [BlockHash(bytes([i + 1]) * 4) for i in range(5)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
send_thread._maybe_offload_boundary_states(req)
|
||||
|
||||
worker.store = store
|
||||
|
||||
# A 13-token prompt sharing the prefix must hit the stored boundary at 12.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 12
|
||||
|
||||
|
||||
def test_worker_setup_tolerates_finer_scratch_group():
|
||||
"""Setup and lookup must tolerate a non-prefix-cacheable scratch group.
|
||||
|
||||
Regression: GLM-5.3-Flash carries a kpool-tail scratch group whose block
|
||||
size (``index_kpool`` tokens) is finer than the hash unit, so the
|
||||
coordinator's divisibility assert and the scratch group's
|
||||
``ChunkedTokenDatabase`` both rejected worker setup for any hash unit the
|
||||
scratch block does not divide. Scratch groups never participate in
|
||||
store/load/lookup, so setup must skip them and lookup must still hit
|
||||
stored sub-block boundaries on the participating groups.
|
||||
"""
|
||||
full = FullAttentionSpec(block_size=16, num_kv_heads=8, head_size=64, dtype=None)
|
||||
mamba = MambaSpec(
|
||||
block_size=16,
|
||||
shapes=((1, 1),),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
)
|
||||
# Scratch block 4 is not divisible by the hash unit 8 below.
|
||||
scratch = KpoolTailSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=64,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
cfg = KVCacheConfig(
|
||||
num_blocks=4,
|
||||
kv_cache_tensors=[
|
||||
KVCacheTensor(
|
||||
size=4 * full.page_size_bytes,
|
||||
layers=["L0"],
|
||||
layer_stride=4 * full.page_size_bytes,
|
||||
block_stride=full.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * mamba.page_size_bytes,
|
||||
layers=["L1"],
|
||||
layer_stride=4 * mamba.page_size_bytes,
|
||||
block_stride=mamba.page_size_bytes,
|
||||
),
|
||||
KVCacheTensor(
|
||||
size=4 * scratch.page_size_bytes,
|
||||
layers=["L2"],
|
||||
layer_stride=4 * scratch.page_size_bytes,
|
||||
block_stride=scratch.page_size_bytes,
|
||||
),
|
||||
],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(["L0"], full),
|
||||
KVCacheGroupSpec(["L1"], mamba),
|
||||
KVCacheGroupSpec(["L2"], scratch),
|
||||
],
|
||||
)
|
||||
vllm_config = _minimal_vllm_config(cache_block_size=16)
|
||||
# Hash unit 8 divides the participating groups (16) but not the scratch
|
||||
# group (4); mamba block 16 > 8 keeps partial hash hits on.
|
||||
vllm_config.cache_config.prefix_match_unit = 8
|
||||
store = _DictStore()
|
||||
|
||||
worker = _build_worker_with_dict_store(vllm_config, cfg, store)
|
||||
worker.tp_size = 1
|
||||
worker.pp_size = 1
|
||||
worker.num_kv_head = 8
|
||||
assert worker.coord.enable_partial_hash_hits
|
||||
# The scratch DB is keyed at its own block size and never probed.
|
||||
assert worker.token_dbs[2].hash_block_size == 4
|
||||
|
||||
for g_idx, db in enumerate(worker.token_dbs):
|
||||
db.set_kv_caches_base_addr([g_idx * 10_000])
|
||||
db.set_block_len([512])
|
||||
|
||||
send_thread = KVCacheStoreSendingThread(
|
||||
store=store,
|
||||
token_databases=worker.token_dbs,
|
||||
block_size=worker.block_size,
|
||||
coord=worker.coord,
|
||||
tp_rank=0,
|
||||
group_put_steps=[1, 1, 1],
|
||||
kv_role="kv_both",
|
||||
ready_event=threading.Event(),
|
||||
replicate_config=MagicMock(),
|
||||
group_participates=[True, True, False],
|
||||
)
|
||||
|
||||
# Persist the sub-block partial tail at boundary 12 (keyed by hs[12//8-1]).
|
||||
hs = [BlockHash(bytes([i + 1]) * 8) for i in range(3)]
|
||||
req = ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=0,
|
||||
block_ids=([1], [2], [3]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
num_prompt_tokens=20,
|
||||
boundary_state_offloads=[(1, 7, 12)],
|
||||
)
|
||||
send_thread._maybe_offload_boundary_states(req)
|
||||
|
||||
worker.store = store
|
||||
|
||||
# A 13-token prompt sharing the prefix must hit the first hash unit.
|
||||
assert worker.lookup(num_tokens=13, block_hashes=hs).hit_length == 8
|
||||
# The scratch group's namespace never enters the store.
|
||||
scratch_prefix = worker.token_dbs[2].key_for(hs[0]).rsplit("@", 1)[0]
|
||||
assert not any(key.startswith(scratch_prefix) for key in store._data)
|
||||
|
||||
@@ -211,6 +211,167 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
|
||||
return worker
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
@pytest.mark.parametrize("logical_block_size", [1152, 640])
|
||||
@pytest.mark.parametrize("tail_first", [False, True])
|
||||
def test_register_compressed_indexer_uses_virtual_transfer_pages(
|
||||
logical_block_size, tail_first
|
||||
):
|
||||
"""Compressed indexer rows must split into contiguous NIXL transfer pages."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from vllm.config import set_current_vllm_config
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
|
||||
base_worker as bw,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
|
||||
NixlConnectorWorker,
|
||||
)
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheLayout,
|
||||
KVCacheTensor,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
|
||||
num_logical_blocks = 3
|
||||
transfer_block_size = 64
|
||||
kernel_block_size = 128
|
||||
tokens_per_state = 4
|
||||
state_content_bytes = 132
|
||||
|
||||
indexer_spec = MLAAttentionSpec(
|
||||
block_size=logical_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.uint8,
|
||||
state_content_bytes=state_content_bytes,
|
||||
tokens_per_state=tokens_per_state,
|
||||
)
|
||||
indexer_page_size = indexer_spec.page_size_bytes
|
||||
tail_spec = KpoolTailSpec(
|
||||
block_size=tokens_per_state,
|
||||
num_kv_heads=2,
|
||||
head_size=128,
|
||||
head_size_v=0,
|
||||
dtype=torch.bfloat16,
|
||||
page_size_padded=indexer_page_size,
|
||||
sliding_window=tokens_per_state,
|
||||
)
|
||||
|
||||
allocation_size = num_logical_blocks * indexer_page_size
|
||||
indexer_tensor = KVCacheTensor(
|
||||
size=allocation_size,
|
||||
layers=["indexer"],
|
||||
layer_stride=allocation_size,
|
||||
block_stride=indexer_page_size,
|
||||
)
|
||||
tail_tensor = KVCacheTensor(
|
||||
size=allocation_size,
|
||||
layers=["tail"],
|
||||
layer_stride=allocation_size,
|
||||
block_stride=indexer_page_size,
|
||||
)
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=num_logical_blocks,
|
||||
kv_cache_tensors=[indexer_tensor, tail_tensor],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["indexer"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=logical_block_size,
|
||||
kv_cache_specs={"indexer": indexer_spec},
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["tail"],
|
||||
UniformTypeKVCacheSpecs(
|
||||
block_size=tokens_per_state,
|
||||
kv_cache_specs={"tail": tail_spec},
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
|
||||
raw = torch.zeros(allocation_size, dtype=torch.int8)
|
||||
(indexer_cache,) = create_kv_cache_views(
|
||||
raw,
|
||||
indexer_spec,
|
||||
num_logical_blocks,
|
||||
KVCacheLayout.LBHNC,
|
||||
indexer_tensor,
|
||||
kernel_block_size=kernel_block_size,
|
||||
)
|
||||
(tail_cache,) = create_kv_cache_views(
|
||||
raw,
|
||||
tail_spec,
|
||||
num_logical_blocks,
|
||||
KVCacheLayout.LBHNC,
|
||||
tail_tensor,
|
||||
)
|
||||
assert indexer_cache.data_ptr() == tail_cache.data_ptr() == raw.data_ptr()
|
||||
|
||||
vllm_config = create_vllm_config(block_size=logical_block_size)
|
||||
vllm_config.cache_config.kv_cache_layout = "LBHNC"
|
||||
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
|
||||
fake_backend = MagicMock()
|
||||
fake_backend.get_supported_kernel_block_sizes.return_value = [transfer_block_size]
|
||||
fake_backend.get_name.return_value = "DEEPSEEK_V32_INDEXER"
|
||||
fake_backend.full_cls_name.return_value = "fake.DEEPSEEK_V32_INDEXER"
|
||||
fake_platform = MagicMock()
|
||||
fake_platform.device_type = "cuda"
|
||||
fake_platform.get_nixl_memory_type.return_value = "VRAM"
|
||||
|
||||
caches = [("indexer", indexer_cache), ("tail", tail_cache)]
|
||||
if tail_first:
|
||||
caches.reverse()
|
||||
|
||||
with (
|
||||
patch.object(bw, "NixlWrapper", _RecordingNixl),
|
||||
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
|
||||
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
|
||||
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
|
||||
patch.object(bw, "current_platform", fake_platform),
|
||||
set_current_vllm_config(vllm_config),
|
||||
):
|
||||
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
|
||||
worker.use_mla = True
|
||||
worker.register_kv_caches(dict(caches))
|
||||
|
||||
transfer_page_size = transfer_block_size // tokens_per_state * state_content_bytes
|
||||
num_transfer_blocks = num_logical_blocks * (
|
||||
logical_block_size // transfer_block_size
|
||||
)
|
||||
expected_descs = np.asarray(
|
||||
[
|
||||
[
|
||||
raw.data_ptr() + block_idx * transfer_page_size,
|
||||
transfer_page_size,
|
||||
0,
|
||||
]
|
||||
for block_idx in range(num_transfer_blocks)
|
||||
],
|
||||
dtype=np.uint64,
|
||||
)
|
||||
|
||||
assert worker.block_size == transfer_block_size
|
||||
assert worker.num_regions == 1
|
||||
assert worker.block_len_per_layer == [transfer_page_size]
|
||||
assert worker.block_stride_per_layer == [transfer_page_size]
|
||||
assert worker._region_is_mla == [True]
|
||||
assert worker.kv_caches_base_addr[worker.engine_id][0] == [raw.data_ptr()]
|
||||
assert worker._registered_descs[0] == [(raw.data_ptr(), raw.nbytes, 0, "")]
|
||||
np.testing.assert_array_equal(worker.src_blocks_data, expected_descs)
|
||||
assert expected_descs[-1, 0] + expected_descs[-1, 1] == (
|
||||
raw.data_ptr() + raw.nbytes
|
||||
)
|
||||
|
||||
|
||||
def _make_remote_meta(
|
||||
worker,
|
||||
remote_block_size,
|
||||
|
||||
@@ -143,15 +143,22 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("kernel_block_sizes", "expected_num_blocks", "expected_num_states"),
|
||||
(
|
||||
"kernel_block_sizes",
|
||||
"storage_block_size",
|
||||
"expected_num_blocks",
|
||||
"expected_num_states",
|
||||
),
|
||||
[
|
||||
(None, 4, 64),
|
||||
([256], 4, 64),
|
||||
([64], 16, 16),
|
||||
(None, None, 4, 64),
|
||||
([256], None, 4, 64),
|
||||
([64], None, 16, 16),
|
||||
([64], 256, 4, 64),
|
||||
],
|
||||
)
|
||||
def test_allocate_compressed_mla_cache(
|
||||
kernel_block_sizes: list[int] | None,
|
||||
storage_block_size: int | None,
|
||||
expected_num_blocks: int,
|
||||
expected_num_states: int,
|
||||
):
|
||||
@@ -161,6 +168,7 @@ def test_allocate_compressed_mla_cache(
|
||||
head_size=128,
|
||||
dtype=torch.bfloat16,
|
||||
tokens_per_state=4,
|
||||
storage_block_size=storage_block_size,
|
||||
)
|
||||
num_pages = 4
|
||||
config = KVCacheConfig(
|
||||
|
||||
@@ -76,6 +76,7 @@ def test_packed_dsv4_zeroer_zeroes_only_each_layers_page():
|
||||
static_forward_context={
|
||||
f"layer.{i}": SimpleNamespace(kv_cache=views[i]) for i in range(NUM_LAYERS)
|
||||
},
|
||||
num_blocks=NUM_BLOCKS,
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
@@ -153,6 +154,7 @@ def test_overlaid_zeroer_dedups_segments_with_max_span():
|
||||
static_forward_context={
|
||||
name: SimpleNamespace(kv_cache=views[name]) for name in views
|
||||
},
|
||||
num_blocks=config.num_blocks,
|
||||
)
|
||||
seg_addrs, seg_block_strides, seg_page_sizes, _, _, n_segs = zeroer._meta
|
||||
|
||||
|
||||
@@ -54,6 +54,7 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
static_forward_context={
|
||||
layer_name: SimpleNamespace(kv_cache=storage),
|
||||
},
|
||||
num_blocks=4,
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([1])
|
||||
@@ -64,6 +65,56 @@ def test_attention_blocks_are_zeroed(spec):
|
||||
assert torch.equal(storage, expected)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_layers_in_one_group_may_use_different_kernel_pages_per_block():
|
||||
"""Derive kernel pages per block from each layer's allocation."""
|
||||
device = torch.device("cuda")
|
||||
num_blocks = 4
|
||||
spec = SlidingWindowSpec(
|
||||
block_size=2,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.int32,
|
||||
sliding_window=2,
|
||||
)
|
||||
# Two kernel pages per logical block.
|
||||
wide = torch.ones((num_blocks * 2, 3), dtype=torch.int32, device=device)
|
||||
# One kernel page per logical block, carved out of a larger allocation so
|
||||
# an over-strided write lands in the guard region instead of faulting or
|
||||
# silently hitting another tensor.
|
||||
narrow_backing = torch.ones((num_blocks * 3, 5), dtype=torch.int32, device=device)
|
||||
narrow = narrow_backing[:num_blocks]
|
||||
|
||||
zeroer = KVBlockZeroer(
|
||||
device,
|
||||
attn_groups_iter=[
|
||||
AttentionGroup(
|
||||
None,
|
||||
["wide", "narrow"],
|
||||
spec,
|
||||
0,
|
||||
)
|
||||
],
|
||||
kernel_block_sizes=[1],
|
||||
static_forward_context={
|
||||
"wide": SimpleNamespace(kv_cache=wide),
|
||||
"narrow": SimpleNamespace(kv_cache=narrow),
|
||||
},
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
|
||||
zeroer.zero_block_ids([num_blocks - 1])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected_wide = torch.ones_like(wide)
|
||||
expected_wide[2 * (num_blocks - 1) :] = 0
|
||||
assert torch.equal(wide, expected_wide)
|
||||
|
||||
expected_narrow = torch.ones_like(narrow_backing)
|
||||
expected_narrow[num_blocks - 1] = 0
|
||||
assert torch.equal(narrow_backing, expected_narrow)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")
|
||||
def test_block_ids_are_not_overwritten_while_copy_is_in_flight():
|
||||
device = torch.device("cuda")
|
||||
@@ -207,6 +258,7 @@ def test_large_dsv4_launch_geometry(monkeypatch):
|
||||
name: SimpleNamespace(kv_cache=storage)
|
||||
for name, storage in storages.items()
|
||||
},
|
||||
num_blocks=1,
|
||||
)
|
||||
|
||||
assert zeroer._meta is not None
|
||||
@@ -332,6 +384,7 @@ def test_zeroes_exactly_one_block_per_layer(layout: KVCacheLayout):
|
||||
attn_groups_iter=iter(groups),
|
||||
kernel_block_sizes=[spec.block_size],
|
||||
static_forward_context=ctx,
|
||||
num_blocks=num_blocks,
|
||||
)
|
||||
zeroer.zero_block_ids([2])
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
@@ -7,15 +7,57 @@ from unittest.mock import Mock
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.compilation import CUDAGraphMode
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.v1.attention.backends.recoverssm_metadata import (
|
||||
RecoverSSMMetadata,
|
||||
RecoverSSMPostprocessMetadata,
|
||||
)
|
||||
from vllm.v1.worker.gpu.model_states import mamba_hybrid
|
||||
from vllm.v1.worker.gpu.model_states.mamba_hybrid import MambaHybridModelState
|
||||
from vllm.v1.worker.gpu.model_states.recoverssm import RecoverSSMState
|
||||
|
||||
|
||||
def test_prepare_attn_forwards_positions(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
state = object.__new__(MambaHybridModelState)
|
||||
state.vllm_config = SimpleNamespace(num_speculative_tokens=0)
|
||||
state.max_model_len = 8192
|
||||
state._align_mode = False
|
||||
state.recoverssm = None
|
||||
|
||||
positions = torch.tensor([1536], dtype=torch.int64)
|
||||
input_batch = SimpleNamespace(
|
||||
num_reqs=1,
|
||||
num_tokens=1,
|
||||
num_reqs_after_padding=1,
|
||||
num_tokens_after_padding=1,
|
||||
query_start_loc_np=torch.tensor([0, 1], dtype=torch.int32).numpy(),
|
||||
query_start_loc=torch.tensor([0, 1], dtype=torch.int32),
|
||||
num_scheduled_tokens=torch.tensor([1], dtype=torch.int32),
|
||||
seq_lens_cpu_upper_bound=torch.tensor([1537], dtype=torch.int32),
|
||||
seq_lens=torch.tensor([1537], dtype=torch.int32),
|
||||
is_prefilling_np=torch.tensor([False]).numpy(),
|
||||
dcp_local_seq_lens=None,
|
||||
positions=positions,
|
||||
prompt_lens=torch.tensor([1024], dtype=torch.int32),
|
||||
)
|
||||
expected_metadata = {"layer": object()}
|
||||
build_attn_metadata = Mock(return_value=expected_metadata)
|
||||
monkeypatch.setattr(mamba_hybrid, "build_attn_metadata", build_attn_metadata)
|
||||
|
||||
metadata = state.prepare_attn(
|
||||
input_batch=input_batch,
|
||||
cudagraph_mode=CUDAGraphMode.NONE,
|
||||
block_tables=(),
|
||||
slot_mappings=torch.empty(0, dtype=torch.int64),
|
||||
attn_groups=[],
|
||||
kv_cache_config=Mock(),
|
||||
)
|
||||
|
||||
assert metadata is expected_metadata
|
||||
assert build_attn_metadata.call_args.kwargs["positions"] is positions
|
||||
|
||||
|
||||
@pytest.mark.skipif(not current_platform.is_cuda(), reason="Requires CUDA")
|
||||
@pytest.mark.parametrize(("num_sampled", "expected_value"), [(0, 1), (3, 3)])
|
||||
def test_postprocess_state_scalar_with_int32_mapping(
|
||||
|
||||
Reference in New Issue
Block a user