[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
@@ -313,3 +313,49 @@ def test_deepgemm_fp8_fp4_paged_mqa_logits(batch_size: int, next_n: int):
ref_logits = ref_logits.masked_fill(~mask, 0)
diff = calc_diff(logits, ref_logits)
assert diff < 1e-3, f"{diff=}"
@pytest.mark.skipif(not current_platform.is_cuda(), reason="CUDA only")
@pytest.mark.skipif(not has_deep_gemm(), reason="DeepGEMM not available")
@pytest.mark.skipif(
not current_platform.has_device_capability(90), reason="SM90 and SM100 only"
)
def test_deepgemm_paged_mqa_packed_manager_block_stride():
torch.manual_seed(0)
num_blocks, block_size, packed_rows = 4, 256, 320
num_heads, head_dim, row_bytes = 64, 128, 132
page_size, pages_per_block = 64, block_size // 64
dense_stride, packed_stride = pages_per_block, packed_rows // page_size
compact_pages = kv_cache_cast_to_fp8(
torch.randn(
num_blocks, block_size, 1, head_dim, device="cuda", dtype=torch.bfloat16
).view(num_blocks * pages_per_block, page_size, 1, head_dim)
)
packed_backing = torch.zeros(
num_blocks,
packed_rows,
row_bytes,
device="cuda",
dtype=torch.uint8,
)
packed_manager = packed_backing[:, :block_size]
packed_manager.copy_(compact_pages.squeeze(2).view(num_blocks, block_size, -1))
packed_pages = packed_backing.view(-1, page_size, row_bytes).unsqueeze(2)
offsets = torch.arange(pages_per_block, device="cuda", dtype=torch.int32)
tables = [
(stride + offsets).unsqueeze(0) for stride in (dense_stride, packed_stride)
]
q = torch.randn(1, 1, num_heads, head_dim, device="cuda", dtype=torch.bfloat16).to(
torch.float8_e4m3fn
)
weights = torch.randn(1, num_heads, device="cuda", dtype=torch.float32)
context_lens = torch.full((1, 1), block_size, device="cuda", dtype=torch.int32)
schedule = get_paged_mqa_logits_metadata(context_lens, page_size, get_num_sms())
logits = [
fp8_fp4_paged_mqa_logits(
(q, None), cache, weights, context_lens, table, schedule, block_size, False
)
for cache, table in zip((compact_pages, packed_pages), tables)
]
torch.testing.assert_close(*logits, rtol=0, atol=0)
@@ -463,7 +463,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
set_random_seed(412)
num_tokens, num_heads, block_size = 2, 8, 16
q = torch.randn(num_tokens, num_heads, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2 * block_size, 1, Q_HEAD_DIM, device="cuda").to(dtype)
kv = torch.randn(2, 2, block_size, Q_HEAD_DIM, device="cuda").to(dtype)[:, 1]
selected = torch.tensor([[1, 7, 18], [0, 3, 20]], dtype=torch.int32, device="cuda")
sinks = torch.linspace(-3.0, 5.0, num_heads, device="cuda")
impl = object.__new__(ROCMAiterMLASparseImpl)
@@ -504,7 +504,7 @@ def test_sparse_mla_sink_forward_mqa_preserves_split_query(dtype):
SimpleNamespace(_q_scale=None, _k_scale=None),
)
references = [
_sink_reference(q[i], kv[:, 0][selected[i].long()], sinks, SM_SCALE)
_sink_reference(q[i], kv.flatten(0, 1)[selected[i].long()], sinks, SM_SCALE)
for i in range(num_tokens)
]
expected, _, value_scale, score_error = (
@@ -553,12 +553,10 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
owner = ComputeSlotMappingKernel()
compile_key = owner.CompileKey(
kv_cache_block_size=16,
blocks_per_kv_block=1,
total_cp_world_size=2,
total_cp_rank=1,
cp_kv_cache_interleave_size=1,
block_table_stride=128,
block_size=16,
)
launches: list[tuple[Any, ...]] = []
@@ -574,7 +572,6 @@ def test_compute_slot_mapping_uses_named_launcher_inputs(monkeypatch) -> None:
assert inputs["block_table_stride"] == 128
assert kwargs == {
"KV_CACHE_BLOCK_SIZE": 16,
"BLOCKS_PER_KV_BLOCK": 1,
"TOTAL_CP_WORLD_SIZE": 2,
"TOTAL_CP_RANK": 1,
"CP_KV_CACHE_INTERLEAVE_SIZE": 1,
@@ -425,7 +425,7 @@ def _skip_if_backend_unavailable(backend_cls, kv_cache_dtype: str, block_size: i
and kv_cache_dtype != "fp8_ds_mla"
):
pytest.skip("FlashMLA Sparse fp8 only supports fp8_ds_mla kv-cache dtype")
if block_size not in backend_cls.get_supported_kernel_block_sizes():
if not backend_cls.supports_block_size(block_size):
pytest.skip(
f"{backend_cls.get_name()} does not support block_size={block_size}"
)
@@ -29,7 +29,7 @@ HEAD = 512
TOPK = 128 # triton convert requires width % 128 == 0
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **kw):
out = torch.full_like(token_indices, -1)
counts = torch.zeros(token_indices.shape[0], dtype=torch.int32)
for t in range(token_indices.shape[0]):
@@ -41,7 +41,7 @@ def ref_convert(req_id, block_table, token_indices, BLOCK_SIZE=64, **_):
blk = int(block_table[int(req_id[t]), pos // BLOCK_SIZE])
if blk < 0:
continue
vals.append(blk * BLOCK_SIZE + pos % BLOCK_SIZE)
vals.append(blk * kw["BLOCK_STRIDE_ROWS"] + pos % BLOCK_SIZE)
out[t, : len(vals)] = torch.tensor(vals, dtype=out.dtype)
counts[t] = len(vals)
return out, counts
@@ -133,11 +133,8 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
meta.state = state
q_nope = torch.randn(rows, impl.num_heads, HEAD)
q_rope = torch.randn(rows, impl.num_heads, qk_rope)
cache = torch.zeros(
8 * BLOCK_SIZE,
impl.head_size,
dtype=torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16,
)
dtype = torch.uint8 if impl.use_fp8_kv_cache else torch.bfloat16
cache = torch.zeros(8, 2, BLOCK_SIZE, impl.head_size, dtype=dtype)[:, 0]
out, lse = impl.forward_mqa(
(q_nope, q_rope), cache, meta, SimpleNamespace(_k_scale_float=0.5)
@@ -147,7 +144,10 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
# Reserved buffers carry this step's slots; lengths are NOT refreshed
# here (the builder plans them host-side before capture/replay).
ref_slots, ref_counts = ref_convert(
meta.req_id_per_token, meta.block_table, impl.topk_indices_buffer
meta.req_id_per_token,
meta.block_table,
impl.topk_indices_buffer,
BLOCK_STRIDE_ROWS=2 * BLOCK_SIZE,
)
offset = 0
for t in range(rows):
@@ -161,7 +161,7 @@ def test_forward_wiring(monkeypatch, qk_rope, kv_dtype):
assert state.wrapper.run_args is not None
q_pe, ckv, kpe, kwargs = state.wrapper.run_args[1:]
assert q_pe.shape == (rows, impl.num_heads, qk_rope)
assert ckv.shape == (8 * BLOCK_SIZE, 1, HEAD)
assert ckv.shape == ((8 - 1) * 2 * BLOCK_SIZE + BLOCK_SIZE, 1, HEAD)
assert kpe.shape[-1] == qk_rope
if impl.use_fp8_kv_cache:
assert kwargs["ckv_scale"] == 0.5 and kwargs["kpe_scale"] == 1.0
@@ -24,6 +24,21 @@ from vllm.platforms.interface import DeviceCapability
from vllm.v1.attention.backends.mla.flashmla_sparse import (
QUANTIZED_DS_MLA_CACHE_FORMATS,
FlashMLASparseBackend,
_fp8_kv_pages,
)
from vllm.v1.kv_cache_interface import (
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
KVQuantMode,
MLAAttentionSpec,
)
from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
prepare_kernel_block_sizes,
select_common_block_size,
)
SM90 = DeviceCapability(major=9, minor=0)
@@ -83,6 +98,88 @@ def test_supported_head_sizes_include_512():
assert FlashMLASparseBackend.get_supported_head_sizes() == [576, 512]
def test_flashmla_bf16_nope_accepts_packed_manager_blocks():
spec = MLAAttentionSpec(
block_size=1152,
num_kv_heads=1,
head_size=512,
dtype=torch.bfloat16,
cache_dtype_str="bfloat16",
block_stride_alignment=1024,
)
num_blocks = 2
layers = ["layer.0", "layer.1"]
page_size = spec.page_size_bytes
config = KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=[
KVCacheTensor(
size=num_blocks * len(layers) * page_size,
layers=layers,
layer_stride=page_size,
block_stride=len(layers) * page_size,
)
],
kv_cache_groups=[KVCacheGroupSpec(layers, spec)],
)
groups = [[AttentionGroup(FlashMLASparseBackend, layers, spec, 0)]]
kernel_block_sizes = prepare_kernel_block_sizes(config, groups)
caches = allocate_kv_cache(
config,
torch.device("cpu"),
KVCacheLayout.BLHNC,
kernel_block_sizes,
)
assert kernel_block_sizes == [1152]
assert caches["layer.0"].shape == (num_blocks, 1, 1152, 512)
@pytest.mark.parametrize("sm100", [False, True])
def test_flashmla_quantized_cache_kernel_pages(monkeypatch, sm100):
"""Quantized blocks are read as 64-token pages; SM100 only row-aligns them
(TMA) and keeps 64-token blocks."""
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda family, device_id=0: sm100 and family == 100,
)
spec = MLAAttentionSpec(
block_size=1152,
num_kv_heads=1,
head_size=576,
dtype=torch.float8_e4m3fn,
cache_dtype_str="fp8_ds_mla",
kv_quant_mode=KVQuantMode.FP8_PER_TENSOR,
state_content_bytes=656,
)
kernel_block_size = 64 if sm100 else 1152
assert (
select_common_block_size(1152, [FlashMLASparseBackend], [spec])
== kernel_block_size
)
def test_fp8_kv_pages_address_packed_blocks():
"""Token t of block b is row b * block_stride_rows + t of the 64-row pages;
64-token blocks are passed through."""
row, block_size, stride_rows = 656, 128, 192
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
cache = backing.view(3, stride_rows, row)[:, :block_size]
pages, block_stride_rows = _fp8_kv_pages(cache, block_size)
assert block_stride_rows == stride_rows
assert pages.shape[1:] == (64, row)
slot = 2 * block_stride_rows + 100
assert torch.equal(pages[slot // 64, slot % 64], cache[2, 100])
pages, block_stride_rows = _fp8_kv_pages(cache[:, :64], 64)
assert block_stride_rows is None
assert torch.equal(pages, cache[:, :64])
def test_quantized_ds_mla_formats_are_the_envelope_set():
assert frozenset({"fp8_ds_mla", "nvfp4_ds_mla"}) == QUANTIZED_DS_MLA_CACHE_FORMATS
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLM-5.3-Flash indexer kernel block sizes are DeepGEMM pool pages."""
from types import SimpleNamespace
import pytest
from vllm.platforms import current_platform
from vllm.v1.attention.backends.mla import indexer
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
KPOOL = 4
@pytest.mark.parametrize(
"rocm,sm120,expected",
[
(False, False, [KPOOL * 32, KPOOL * 64]),
# SM120 DeepGEMM takes only 64-entry pages for the fp8 cache.
(False, True, [KPOOL * 64]),
# ROCm's AITER paged-MQA logits take the same pool pages.
(True, False, [KPOOL * 32, KPOOL * 64]),
],
)
def test_glm5next_indexer_kernel_block_sizes(monkeypatch, rocm, sm120, expected):
monkeypatch.setattr(current_platform, "is_rocm", lambda: rocm)
monkeypatch.setattr(
current_platform,
"is_device_capability_family",
lambda family: sm120 and family == 120,
)
model_config = SimpleNamespace(hf_text_config=SimpleNamespace(index_kpool=KPOOL))
config = SimpleNamespace(model_config=model_config)
monkeypatch.setattr(indexer, "get_current_vllm_config", lambda: config)
spec = SimpleNamespace(tokens_per_state=KPOOL)
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes(spec) == expected
# Platform block-size selection queries without a spec.
assert Glm5NextIndexerBackend.get_supported_kernel_block_sizes() == expected
@@ -122,7 +122,6 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = 16
builder.kernel_block_size = None
builder.num_sms = num_sms
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(num_states=64)
@@ -170,30 +169,24 @@ def test_indexer_draft_decode_metadata_update_is_capture_safe():
@pytest.mark.skipif(not torch.cuda.is_available(), reason="requires CUDA")
@pytest.mark.parametrize(
"block_size,kernel_block_size,compress_ratio", [(64, 64, 1), (256, 64, 4)]
)
def test_indexer_draft_decode_refresh_follows_new_batch(
monkeypatch, block_size, kernel_block_size, compress_ratio
):
@pytest.mark.parametrize("compress_ratio", [1, 4])
def test_indexer_draft_decode_refresh_follows_new_batch(monkeypatch, compress_ratio):
"""Without a prior build(), the refresh must not keep the last batch's state."""
monkeypatch.setattr(indexer_module, "has_deep_gemm", lambda: False)
device = torch.device("cuda")
num_reqs, width = 3, 8
factor = block_size // kernel_block_size
builder = object.__new__(DeepseekV32IndexerMetadataBuilder)
builder.dcp_world_size = 1
builder.compress_ratio = compress_ratio
builder.kernel_block_size = kernel_block_size
builder.arange_buffer = torch.arange(num_reqs + 1, dtype=torch.int32, device=device)
builder.kv_cache_spec = SimpleNamespace(block_size=block_size, num_states=64)
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
def stale(*shape: int) -> torch.Tensor:
return torch.full(shape, -7, dtype=torch.int32, device=device)
decode = DeepSeekV32IndexerDecodeMetadata(
block_table=stale(num_reqs, width // factor),
block_table=stale(num_reqs, width),
seq_lens=stale(num_reqs, 1),
decode_lens=stale(num_reqs),
requires_padding=False,
@@ -219,8 +212,7 @@ def test_indexer_draft_decode_refresh_follows_new_batch(
builder.update_draft_decode_metadata(metadata)
expected_block_table = request_block_table[:, ::factor] // factor
torch.testing.assert_close(decode.block_table, expected_block_table)
torch.testing.assert_close(decode.block_table, request_block_table)
torch.testing.assert_close(decode.indices, builder.arange_buffer[:num_reqs])
assert torch.all(decode.per_req_decode_lens == 1)
assert torch.all(decode.decode_lens == 1)
@@ -471,7 +463,6 @@ def test_zero_token_pcp_rank_participates_in_compressed_mapping_gather(monkeypat
builder.pcp_rank = 0
builder.dcp_world_size = 1
builder.cp_kv_cache_interleave_size = 1
builder.kernel_block_size = None
builder.kv_cache_spec = SimpleNamespace(block_size=64, num_states=64)
builder.compressed_slot_mapping_buffer = torch.zeros(8, dtype=torch.int64)
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""CPU tests for storage-block kernel-block selection in prepare_kernel_block_sizes."""
"""CPU tests for the kpool indexer's kernel block selection."""
from types import SimpleNamespace
@@ -11,20 +11,14 @@ from vllm.config import VllmConfig, set_current_vllm_config
from vllm.models.glm5next.common.attention import Glm5NextIndexerCache
from vllm.v1.attention.backend import AttentionBackend, MultipleOf
from vllm.v1.attention.backends.mla import indexer as indexer_mod
from vllm.v1.attention.backends.mla.indexer import (
DeepseekV32IndexerBackend,
KpoolTailBackend,
)
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
)
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.utils import prepare_kernel_block_sizes
from vllm.v1.worker.utils import prepare_kernel_block_sizes, select_common_block_size
pytestmark = pytest.mark.cpu_test
# ``index_kpool`` of zai-org/GLM-5.3-Flash: one indexer state pools 4
# compressed states (hence storage_block_size = page * 4).
# compressed states (hence a kernel block is page * 4 tokens).
INDEX_KPOOL = 4
# The hybrid KDA/mamba manager floors at TP8..TP1; 640 is the geometry of the
@@ -59,14 +53,17 @@ def _prepare(
spec: MLAAttentionSpec, backends: list[type[AttentionBackend]]
) -> list[int]:
kv_cache_config = SimpleNamespace(
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec)]
kv_cache_groups=[SimpleNamespace(kv_cache_spec=spec, layer_names=[])],
kv_cache_tensors=[],
)
attn_groups = [[SimpleNamespace(backend=backend) for backend in backends]]
attn_groups = [
[SimpleNamespace(backend=backend, kv_cache_spec=spec) for backend in backends]
]
return prepare_kernel_block_sizes(kv_cache_config, attn_groups)
@pytest.mark.parametrize("manager_block", TP_FLOOR_MANAGER_BLOCKS)
def test_prepare_uses_storage_block_for_the_kpool_group(
def test_kpool_group_maps_pool_pages(
monkeypatch: pytest.MonkeyPatch, manager_block: int
):
"""The bug: on ROCm, kpool groups must get 128/256 - not the 640-4352
@@ -78,54 +75,43 @@ def test_prepare_uses_storage_block_for_the_kpool_group(
"""
_mock_rocm_platform(monkeypatch)
spec = _kpool_storage_spec(manager_block)
expected_storage = 256 if manager_block % 256 == 0 else 128
assert spec.storage_block_size == expected_storage
expected = 256 if manager_block % 256 == 0 else 128
selected = _prepare(
spec,
[ROCMAiterMLASparseBackend, DeepseekV32IndexerBackend, KpoolTailBackend],
)
assert selected == [expected_storage]
# The kpool group's AttentionGroup selects its kernel block this way.
selected = [
select_common_block_size(manager_block, [Glm5NextIndexerBackend], [spec])
]
assert selected == [expected]
assert selected != [manager_block]
# The hybrid block-table split stays integral.
assert manager_block % selected[0] == 0
def test_prepare_falls_back_when_storage_block_is_not_supported():
"""No storage block, or a storage block the backends reject (e.g. the
CUDA indexer's exact ``[64]`` vs a 128-token storage block): selection is
`select_common_block_size`'s answer, unchanged from no-fix behavior.
"""
def test_prepare_uses_the_backend_vote():
"""Unpacked groups take `select_common_block_size`'s answer."""
class Fixed64Backend:
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return [64]
@staticmethod
def get_name() -> str:
return "FIXED64"
# Storage block the group's backend does not accept -> backend vote (64).
spec = _kpool_storage_spec(640)
assert spec.storage_block_size == 128
assert _prepare(spec, [Fixed64Backend]) == [64]
# No storage block at all (any non-kpool MLA model) -> backend vote.
plain_spec = MLAAttentionSpec(
block_size=640,
num_kv_heads=1,
head_size=128,
dtype=torch.bfloat16,
)
assert plain_spec.storage_block_size is None
assert _prepare(plain_spec, [Fixed64Backend]) == [64]
# ...and with backends that accept the manager block, the manager wins,
# exactly as on main.
class AcceptAllBackend:
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return [MultipleOf(1)]
@staticmethod
@@ -3,7 +3,7 @@
"""CPU tests for the kpool tail slot mapping (no GPU required).
The kpool tail cache is a 1-block-per-request circular ring addressed by
``pos % kpool`` (``KpoolTailSpec`` / ``KpoolTailManager``: exactly one block
``pos % kpool`` (``CircularBufferSpec`` / ``CircularBufferManager``: exactly one block
allocated per request, never grown, so only column 0 of its block table is
ever written; the rest stays zero-initialized).
@@ -32,27 +32,25 @@ from vllm.v1.attention.backends.mla.indexer import (
KpoolTailMetadataBuilder,
compute_kpool_tail_slot_mapping,
)
from vllm.v1.kv_cache_interface import KpoolTailSpec, compute_layout_strides
from vllm.v1.kv_cache_interface import CircularBufferSpec, compute_layout_strides
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.block_table import get_block_table_width
KPOOL = 4
def test_tail_backend_layout_matches_kernel_pointer_arithmetic():
(layout,) = KpoolTailBackend.supported_kv_cache_layouts()
spec = KpoolTailSpec(
spec = CircularBufferSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
strides = compute_layout_strides(spec, num_blocks=8, num_layers=3, layout=layout)
_, _, head_stride, state_stride, content_stride = strides
assert layout is KVCacheLayout.LBHNC
assert layout is KVCacheLayout.BLHNC
assert head_stride == KPOOL * 128 * torch.bfloat16.itemsize
assert state_stride == 128 * torch.bfloat16.itemsize
assert content_stride == 1
@@ -77,37 +75,14 @@ def test_tail_ring_divides_the_attention_block(num_speculative_tokens, ring):
SimpleNamespace(num_speculative_tokens=num_speculative_tokens)
)
assert spec.block_size == spec.sliding_window == ring
assert isinstance(spec, CircularBufferSpec) and spec.block_size == ring
assert ring >= KPOOL + num_speculative_tokens
assert 640 % ring == 0
def test_tail_spec_opts_out_of_generic_slot_mapping():
"""The tail row is one block wide (padded to the block-table alignment), so
the generic kernel's ``pos // kpool`` column index runs off the end of the
allocation for long prompts. The spec must opt out of it entirely."""
spec = KpoolTailSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
max_len = 1 << 20
width = get_block_table_width(
spec.max_num_blocks_per_req(None, max_len),
spec.block_size,
token_alignment=spec.block_table_token_alignment,
)
assert width * KPOOL < max_len
assert spec.uses_slot_mapping is False
def make_tail_block_table(own_blocks, width=64):
"""Tail-group block table as BlockTables produces it: column 0 holds the
request's single KpoolTailManager block, the remaining columns are never
request's single CircularBufferManager block, the remaining columns are never
written and stay zero."""
bt = torch.zeros(len(own_blocks), width, dtype=torch.int32)
bt[:, 0] = torch.tensor(own_blocks, dtype=torch.int32)
@@ -447,6 +422,7 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
tokens between the last request boundary and num_actual_tokens (mapped to
the last request), untouched padding beyond num_actual, and PAD for
requests on the null block."""
ring_size = 2 * KPOOL
positions, qsl, slot_mapping, _, num_reqs = make_batch(
per_req, padded_len=padded_len
)
@@ -455,14 +431,17 @@ def test_triton_mapping_matches_cpu(per_req, own_blocks, num_actual, padded_len)
slot_mapping = torch.arange(padded_len, dtype=torch.int64) + 1000
bt = make_tail_block_table(own_blocks)
ref = circular_tail_slots(slot_mapping, bt, qsl, positions, num_actual, num_reqs)
got = circular_tail_slots(
ref = compute_kpool_tail_slot_mapping(
slot_mapping, bt, qsl, positions, num_actual, num_reqs, ring_size
)
got = compute_kpool_tail_slot_mapping(
slot_mapping.cuda(),
bt.cuda(),
qsl.cuda().to(torch.int32),
positions.cuda(),
num_actual,
num_reqs,
ring_size,
)
torch.testing.assert_close(got.cpu(), ref)
for req, blk in enumerate(own_blocks):
-1
View File
@@ -805,7 +805,6 @@ def test_composite_shared_cache_across_image_and_causal_steps(
)
layer.mm_prefix_clamp_sliding_window = True
builder = backend.get_builder_cls()(spec, ["composite_test"], cfg, DEVICE)
builder.set_kernel_block_size(block_size)
# Match production, where unwritten page tails are zero-initialized.
cache = torch.zeros(
num_blocks,
@@ -11,6 +11,7 @@ from vllm.triton_utils import tl, triton
from vllm.utils.torch_utils import set_random_seed
from vllm.v1.attention.backends.mla import rocm_aiter_mla_sparse as sparse_mod
from vllm.v1.attention.backends.mla.rocm_aiter_mla_sparse import (
ROCMAiterMLASparseBackend,
_use_rocm_sparse_triton,
fit_kpool_indices_to_aiter,
)
@@ -19,6 +20,7 @@ from vllm.v1.attention.ops.rocm_aiter_mla_sparse import (
_validate_dsv4_sparse_dims,
_validate_sparse_dims,
)
from vllm.v1.kv_cache_interface import KVCacheLayout
@triton.jit
@@ -104,6 +106,10 @@ def test_fit_kpool_indices_rejects_narrow_input():
)
def test_rocm_sparse_mla_supports_glm_packed_layout():
assert KVCacheLayout.BLHNC in ROCMAiterMLASparseBackend.supported_kv_cache_layouts()
@pytest.mark.parametrize(
(
"kv_cache_dtype",
+65 -4
View File
@@ -140,7 +140,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
metadata = SimpleNamespace(
req_id_per_token=torch.zeros(1, dtype=torch.int32),
block_table=torch.zeros((1, 1), dtype=torch.int32),
block_size=1,
block_size=32,
)
recorded_scale = None
@@ -178,7 +178,7 @@ def test_nope_flashinfer_sparse_mla_uses_model_scale(monkeypatch):
)
impl.forward_mqa(
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 1, kv_lora_rank),
torch.zeros(1, 32, kv_lora_rank),
metadata,
SimpleNamespace(),
)
@@ -437,6 +437,7 @@ def test_sparse_backend_decode_correctness(
qk_rope_head_dim: int,
v_head_dim: int,
monkeypatch,
block_stride_rows: int | None = None,
):
if qk_rope_head_dim == 0 and (
backend_cls != FlashMLASparseBackend
@@ -478,8 +479,7 @@ def test_sparse_backend_decode_correctness(
if device_capability is None or device_capability.major != 10:
pytest.skip("The NVFP4 DS-MLA kv-cache dtype requires SM 10.x")
supported_block_sizes = backend_cls.get_supported_kernel_block_sizes()
if block_size not in supported_block_sizes:
if not backend_cls.supports_block_size(block_size):
pytest.skip(
f"{backend_cls.get_name()} does not support block_size={block_size}"
)
@@ -719,6 +719,11 @@ def test_sparse_backend_decode_correctness(
kv_cache_dtype=kv_cache_dtype,
scale=kv_cache_scale,
)
if block_stride_rows is not None:
pad = (0, 0, 0, block_stride_rows - block_size)
kv_cache = torch.nn.functional.pad(kv_cache, pad)[:, :, :block_size]
slot = int(common_attn_metadata.slot_mapping[-1])
query_row = kv_cache[slot // block_size, 0, slot % block_size].zero_()
# The sparse builder clones the layer's dense-MHA prefill backend from
# static_forward_context; register a mock layer carrying one.
@@ -814,6 +819,12 @@ def test_sparse_backend_decode_correctness(
out_buffer,
)
if block_stride_rows is not None and kv_cache_dtype == "auto":
expected_row = torch.cat((kv_c_vllm[-1], k_pe_vllm[-1, 0]))
torch.testing.assert_close(query_row, expected_row, rtol=0, atol=0)
elif block_stride_rows is not None:
assert query_row.any()
assert backend_output.shape == sdpa_reference.shape
assert backend_output.dtype == sdpa_reference.dtype
assert torch.isfinite(backend_output).all()
@@ -828,6 +839,56 @@ def test_sparse_backend_decode_correctness(
torch.testing.assert_close(backend_output, sdpa_reference, rtol=0.01, atol=0.01)
@pytest.mark.parametrize(
("backend_cls", "kv_cache_dtype", "block_size", "block_stride_rows", "head_dims"),
[
pytest.param(
FlashInferMLASparseTRTLLMBackend,
"auto",
256,
320,
(128, 128, 64, 128),
id="FlashInferTRTLLM",
),
# GLM-5.3 on SM90: the fp8 kernel's 64-token pages in larger blocks.
pytest.param(
FlashMLASparseBackend,
"fp8_ds_mla",
128,
192,
(64, 256, 0, 256),
id="FlashMLA-fp8_ds_mla",
),
],
)
def test_sparse_mla_packed_stride(
default_vllm_config,
dist_init,
workspace_init,
monkeypatch,
backend_cls,
kv_cache_dtype,
block_size,
block_stride_rows,
head_dims,
):
test_sparse_backend_decode_correctness(
default_vllm_config,
dist_init,
backend_cls,
"mixed_small",
kv_cache_dtype,
1,
block_size,
workspace_init,
1.0,
1.0,
*head_dims,
monkeypatch,
block_stride_rows=block_stride_rows,
)
def _triton_convert_reference_impl(
req_ids: torch.Tensor,
block_table: torch.Tensor,
@@ -12,6 +12,7 @@ from vllm.config import CacheConfig
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
from vllm.model_executor.models.deepseek_v2 import DeepseekV32IndexerCache
from vllm.platforms import current_platform
from vllm.v1.attention.backend import MultipleOf
from vllm.v1.attention.backends.mla.flashinfer_mla_sparse import (
FlashInferMLASparseTRTLLMBackend,
)
@@ -25,7 +26,7 @@ from vllm.v1.attention.backends.utils import get_supported_kv_cache_layouts
from vllm.v1.core.kv_cache_utils import get_kv_cache_config_from_groups
from vllm.v1.kv_cache_interface import KVCacheGroupSpec, UniformTypeKVCacheSpecs
from vllm.v1.kv_cache_layout import KVCacheLayout
from vllm.v1.worker.utils import allocate_kv_cache
from vllm.v1.worker.utils import allocate_kv_cache, customize_attention_spec
pytestmark = pytest.mark.skip_global_cleanup
@@ -48,10 +49,10 @@ pytestmark = pytest.mark.skip_global_cleanup
@pytest.mark.parametrize(
"backend,cache_dtype,sm100,stride_alignment",
[
(FlashInferMLASparseTRTLLMBackend, "auto", True, 1152),
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 576),
(FlashInferMLASparseTRTLLMBackend, "auto", True, 32 * 1152),
(FlashInferMLASparseTRTLLMBackend, "fp8", True, 32 * 576),
(FlashMLASparseBackend, "auto", True, 1152),
(FlashMLASparseBackend, "fp8_ds_mla", False, None),
(FlashMLASparseBackend, "fp8_ds_mla", False, MultipleOf(64)),
(FlashMLASparseBackend, "fp8_ds_mla", True, 656),
(FlashMLASparseBackend, "nvfp4_ds_mla", True, 352),
],
@@ -79,7 +80,9 @@ def test_allocation_and_warmup_follow_addressing_mode(
non_causal_multi_token_decode=False,
)
layer._uses_flat_kv_cache = MethodType(MLAAttention._uses_flat_kv_cache, layer)
spec = MLAAttention.get_kv_cache_spec(layer, config)
spec = customize_attention_spec(
backend, MLAAttention.get_kv_cache_spec(layer, config)
)
indexer = SimpleNamespace(
cache_config=config.cache_config, head_dim=132, dtype=torch.uint8
)
@@ -102,10 +105,8 @@ def test_allocation_and_warmup_follow_addressing_mode(
MLAAttention.bind_kv_cache(layer, views["mla"])
DeepseekV32IndexerCache.bind_kv_cache(indexer, views["indexer"])
assert indexer.kv_cache.stride(0) == views["indexer"].stride(0)
if stride_alignment is not None:
assert (
views["mla"].stride(0) * views["mla"].element_size() % stride_alignment == 0
)
alignment = spec.get_block_stride_alignment()
assert views["mla"].stride(0) * views["mla"].element_size() % alignment == 0
if not layer._uses_flat_kv_cache():
register.assert_not_called()
else:
@@ -113,7 +114,7 @@ def test_allocation_and_warmup_follow_addressing_mode(
register.assert_called_once_with(config, block_stride_rows=stride)
layer.kv_cache[1, 0].fill_(3)
torch.testing.assert_close(rows[stride], layer.kv_cache[1, 0])
if stride_alignment is None:
if alignment == 1:
assert cache.kv_cache_tensors[0].size == cache.num_blocks * sum(
item.page_size_bytes for item in specs.values()
)
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=mamba_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=mamba_block_size,
),
),
],
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=kpool_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=kpool_block_size,
),
),
],
+32 -10
View File
@@ -88,7 +88,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
alignment = lcm(1152, hot.page_size_bytes)
stride = _pool_bytes_per_block(groups)
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
assert stride == alignment
layout = KVCacheLayout.BLHNC
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
@@ -254,7 +254,7 @@ class TestCSALinearGrouping:
def test_every_group_fits_one_packed_block(self):
config = _shared_layout_config()
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
pages = _pages(groups)
for group in groups:
@@ -418,7 +418,7 @@ class TestCSALinearGrouping:
groups = _get_packed_kv_cache_groups(config, specs)
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
assert _get_kv_cache_bytes_per_block(groups) == sum(
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
)
# That block holds every GDN state at once, so the repeat pattern alone
@@ -465,7 +465,7 @@ class TestSlidingWindowBucketCap:
pages = _pages(groups)
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
main_bytes = sum(pages[n] for n in main.layer_names)
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
per_group = main_bytes // pages["layers.0.swa"]
assert len(swa_groups) == -(-43 // per_group)
@@ -479,9 +479,9 @@ class TestSlidingWindowBucketCap:
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
groups
)
assert _get_kv_cache_bytes_per_block(
groups, KVCacheLayout.BLHNC
) == _expected_bytes_per_block(groups)
config = get_kv_cache_config_from_groups(
_mock_vllm_config("BLHNC"), groups, MEMORY
@@ -554,6 +554,20 @@ class TestDensePacking:
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
assert mla_tensor.block_stride == block_stride
def test_layer_outer_pads_only_unaligned_pages(self):
"""Layer-outer block strides are pages, so only an unaligned page pads."""
mla_page = _mla(512).page_size_bytes
idx = replace(_mla(128), block_stride_alignment=12288)
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
config = get_kv_cache_config_from_groups(
_mock_vllm_config("LBNHC"), groups, MEMORY
)
mla_tensor, idx_tensor = config.kv_cache_tensors
assert config.num_blocks == MEMORY // (mla_page + 12288)
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
assert idx_tensor.offset == mla_page * config.num_blocks
_bind(config, "LBNHC")
def test_overlaid_groups_alias_and_stay_isolated(self):
groups, g1, g2 = _mixed_page_groups()
# Overlay models resolve to a block-outer layout at backend selection (the
@@ -562,7 +576,9 @@ class TestDensePacking:
_mock_vllm_config("BLNHC"), groups, MEMORY
)
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(
_mock_vllm_config("BLNHC"), groups
) == _expected_bytes_per_block(groups)
views = _bind(config, "BLNHC")
assert set(views) == set(g1) | set(g2)
@@ -667,7 +683,10 @@ class TestCompressorRingGroup:
assert len(ring_groups) == 1
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
manager = KVCacheManager(
@@ -716,7 +735,10 @@ class TestSWABoundedReplayGrouping:
for g in groups
) == [(False, 128)] * 4 + [(True, 0)]
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([kv_cache_config]),
+57 -545
View File
@@ -10,7 +10,7 @@ from collections.abc import Callable
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from typing import Any, cast
from typing import Any
import pytest
import torch
@@ -19,7 +19,6 @@ import vllm.v1.core.kv_cache_utils as kv_cache_utils
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from vllm.config import (
CacheConfig,
KVTransferConfig,
ModelConfig,
SchedulerConfig,
VllmConfig,
@@ -69,7 +68,6 @@ from vllm.v1.kv_cache_interface import (
HiddenStateCacheSpec,
HiSparseHotSpec,
HiSparseResidentSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
@@ -1942,13 +1940,12 @@ def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
config.cache_config.kv_cache_layout = None
tail = KpoolTailSpec(
tail = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
dcp_sharded=False,
)
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
@@ -2906,564 +2903,77 @@ def test_generate_scheduler_kv_cache_config():
)
def _glm5_like_kv_cache_spec(
mamba_spec_factory=new_mamba_spec,
) -> tuple[dict[str, KVCacheSpec], list[str]]:
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
def test_packed_groups_glm5_like_hybrid():
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=8192))
vllm_config.cache_config.kv_cache_layout = "BLHNC"
kpool, block_size = 4, 1024
kv_cache_spec: dict[str, KVCacheSpec] = {}
mamba_layers = []
for i in range(45):
if i % 4 == 3:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=16,
)
else:
name = f"layers.{i}.linear_attn"
kv_cache_spec[name] = mamba_spec_factory()
mamba_layers.append(name)
return kv_cache_spec, mamba_layers
def _glm5_like_kv_cache_spec_with_tail(
mamba_spec_factory=new_mamba_spec,
) -> dict[str, KVCacheSpec]:
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
is why the fixture drops the tokens_per_state=16 shape used above.
"""
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
for i in range(3, 45, 4):
kpool = 4
kv_cache_spec[f"layers.{i}.indexer"] = replace(
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
tokens_per_state=kpool,
for i in range(11):
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
block_stride_alignment=32 * 576 * 2,
)
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=kpool,
block_stride_alignment=64 * 132,
)
kv_cache_spec[f"layers.{i}.tail"] = CircularBufferSpec(
block_size=kpool,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=kpool,
)
return kv_cache_spec
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
return {
layer_name: tensor
for tensor in kv_cache_config.kv_cache_tensors
for layer_name in tensor.layers
}
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
def test_get_kv_cache_config_balanced_mamba_hybrid():
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
uniform_groups = [
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
]
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
# gets an MLA slot.
assert len(groups) == 5
assert len(uniform_groups) == 1
assert len(uniform_groups[0].layer_names) == 22
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
for k, name in enumerate(mamba_layers):
assert name in mamba_groups[k % 4].layer_names
# Mamba pages are padded up to the MLA page.
for group in mamba_groups:
assert group.kv_cache_spec.page_size_padded == mla_page
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
# Every block id is charged the full per-block sum.
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
assert (
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
== blocks_per_request * bytes_per_block
)
available_memory = bytes_per_block * 100 + 1
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, available_memory
)
assert kv_cache_config.num_blocks == 100
# Every logical layer has a view into one backing allocation. Mamba views
# alias their corresponding MLA slot by using the same byte offset.
assert len(kv_cache_config.kv_cache_tensors) == 56
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
tensors = _tensor_by_layer(kv_cache_config)
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
for i, mla_name in enumerate(mla_layer_names):
mla_tensor = tensors[mla_name]
assert mla_tensor.block_stride == mla_page
for group in mamba_groups:
if i < len(group.layer_names):
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
for i in range(11):
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
assert total_allocated == bytes_per_block * 100
assert 0 <= available_memory - total_allocated < bytes_per_block
assert get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config
) == pytest.approx(100 / blocks_per_request)
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
"""The kpool tail parasitizes the indexer tensors instead of getting its
own: sibling idx/tail tensors paired by layer order, zero standalone tail
bytes, one shared block per request, and no prefix-caching leakage from
the kpool-sized scratch group."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
assert tail_logical_page < idx_page
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
tail_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
and all(
isinstance(spec, KpoolTailSpec)
for spec in group.kv_cache_spec.kv_cache_specs.values()
for i in range(34):
kv_cache_spec[f"layers.{i}.kda"] = MambaSpec(
shapes=((3, 6144), (16, 128, 128)),
dtypes=(torch.bfloat16, torch.float32),
block_size=block_size,
)
)
# The tail never prefix-caches; its page is padded up to the indexer page
# so the runner's strided view rides the indexer storage.
assert not tail_group.kv_cache_spec.prefix_cacheable
tail_inner = cast(
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
)
assert tail_inner.page_size_padded == idx_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
# Tail layers get logical views but no additional storage: each view aliases
# its sibling indexer at the same offset in the shared allocation.
assert len(kv_cache_config.kv_cache_tensors) == 67
tensors = _tensor_by_layer(kv_cache_config)
for i in range(11):
idx_name = f"layers.{4 * i + 3}.indexer"
tail_name = f"layers.{4 * i + 3}.tail"
assert tensors[idx_name].offset == tensors[tail_name].offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
# The layout detector surfaces the sibling names in layer order for the
# accounting and connector paths.
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
assert layout is not None
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
# Accounting charges exactly one extra shared block per request for the
# tail, not a per-sequence allocation.
attn_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
and group is not tail_group
)
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
assert (
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
== blocks_per_request * bytes_per_block
)
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
prefix-cache hash granularity: participating groups alone decide it."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
def align_mamba():
return new_mamba_spec(mamba_cache_mode="align")
groups = kv_cache_utils.get_kv_cache_groups(
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
hash_vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
kv_transfer_config=object(),
)
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
# the lcm including the tail (1024 % 4 == 0, so it coincides).
assert kv_cache_utils.resolve_kv_cache_block_sizes(
kv_cache_config, hash_vllm_config
) == (1024, 16)
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
def big_mamba_spec():
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert big_mamba_spec().page_size_bytes > mla_page
with pytest.raises(ValueError, match="does not fit the MLA page"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
"""Use the generic layout error when no kpool indexer is present."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBHNC"
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(27):
if i % 4 == 3 or i == 26:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
shapes=((512, 1024),), dtypes=(torch.float32,)
)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
with pytest.raises(NotImplementedError, match="page size"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
disable slot sharing; the layout re-pads them to the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
def prepadded_mamba_spec():
return new_mamba_spec(page_size_padded=294_912)
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert prepadded_mamba_spec().page_size_bytes < mla_page
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
attn = [
g
for g in groups
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
and isinstance(g.kv_cache_spec.first_spec, MLAAttentionSpec)
]
assert len(groups) == 5
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
for group in mamba_groups:
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
assert len(kv_cache_config.kv_cache_tensors) == 56
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
"""Round-robin mamba grouping keeps practical PP splits balanced: every
stage's largest projected mamba group slice fits its projected MLA
layers, so per-stage slot tensors all have an MLA owner."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
# PP=2 split at transformer layer 22/23.
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
# layers, projected mamba groups [5, 5, 4, 4].
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
global_groups, worker_spec
)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
rings = [
g
for g in groups
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
and isinstance(g.kv_cache_spec.first_spec, CircularBufferSpec)
]
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 5 * mla_page + 5 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
tensors = _tensor_by_layer(kv_cache_config)
# Every projected Mamba slot aliases an MLA owner.
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
# The last slot only hosts the two groups with a 5th projected layer.
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
grouping bumps to 5 groups so every stage's projection keeps sharing
on instead of silently falling back on stage 0."""
from vllm.config import ParallelConfig
from vllm.distributed.utils import get_pp_indices
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=8192),
parallel_config=ParallelConfig(pipeline_parallel_size=4),
)
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
for k, name in enumerate(mamba_layers):
assert name in mamba_groups[k % 5].layer_names
for rank in range(4):
start, end = get_pp_indices(45, rank, 4)
worker_spec = {
n: s
for n, s in kv_cache_spec.items()
if start <= int(n.split(".")[1]) < end
}
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
groups, worker_spec
mamba = [
g
for g in groups
if isinstance(
getattr(g.kv_cache_spec, "first_spec", g.kv_cache_spec), MambaSpec
)
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
from vllm.config import ParallelConfig
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=8192),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
"""Hybrid slot sharing must take precedence over the experimental
enable_cross_layers_blocks packed layout: generic packing would give the
MLA slots a strided view, breaking the contiguous virtual split."""
model_config = ModelConfig(max_model_len=8192)
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
)
vllm_config = VllmConfig(
model_config=model_config, kv_transfer_config=kv_transfer_config
)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
assert len(kv_cache_config.kv_cache_tensors) == 56
assert all(
tensor.block_stride in (mla_page, idx_page)
for tensor in kv_cache_config.kv_cache_tensors
)
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(27):
if i % 4 == 3 or i == 26:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
assert len(groups) == 4
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
for group in mamba_groups:
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 7 * mla_page
assert len(attn) == 1 and len(attn[0].layer_names) == 22
assert len(rings) == 1 and len(rings[0].layer_names) == 11
# 11 * (1.18 MB MLA + 34 KB indexer) per block fits 12 fp32-SSM KDA states.
assert len(mamba) == 3
assert sum(len(g.layer_names) for g in mamba) == 34
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
# Each group has one tensor descriptor and all groups alias the same slots.
assert len(kv_cache_config.kv_cache_tensors) == 4
tensors = _tensor_by_layer(kv_cache_config)
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
for index, mla_name in enumerate(mla_names):
mla_offset = _layer_offset(tensors[mla_name], mla_name)
for group in mamba_groups:
if index < len(group.layer_names):
mamba_name = group.layer_names[index]
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_capacity_after_scheduler_unwrap():
"""max_concurrency must survive the scheduler-config unwrap.
Regression for the balanced-mamba hybrid layout:
``generate_scheduler_kv_cache_config`` flattens the MLA
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(45):
if i % 4 == 3:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
# Confirm we are exercising the regression path: the MLA group is no longer
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
assert not any(
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
for g in scheduler_config.kv_cache_groups
)
unwrapped_groups = scheduler_config.kv_cache_groups
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
vllm_config, unwrapped_groups
)
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
expected_blocks_per_request = (
expected_max_mem + expected_pool - 1
) // expected_pool
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
assert max_concurrency > 0
assert max_concurrency == pytest.approx(
scheduler_config.num_blocks / expected_blocks_per_request
vllm_config, groups, available_memory=2 * 1024**3
)
strides = {t.block_stride for t in kv_cache_config.kv_cache_tensors}
assert len(strides) == 1
(block_stride,) = strides
# Blocks sit a whole number of 32-row MLA pages and 64-state indexer pages
# apart so both caches can be re-paged for their kernels.
assert block_stride % (32 * 576 * 2) == 0
assert block_stride % (64 * 132) == 0
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
@@ -4483,6 +3993,7 @@ def test_kv_cache_reserves_null_block_for_max_model_len(use_override):
block_size = 16
max_model_len = 512 # needs 512 / 16 = 32 blocks
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
vllm_config.cache_config.kv_cache_layout = "LBNHC"
spec = new_kv_cache_spec(block_size=block_size)
# 32 blocks -> only 31 usable after the null block: one short -> reject.
@@ -4528,6 +4039,7 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
block_size = 16
max_model_len = 512 # needs 512 / 16 = 32 blocks
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
vllm_config.cache_config.kv_cache_layout = "LBNHC"
spec = new_kv_cache_spec(block_size=block_size)
# 32 blocks -> only 31 usable after the null block: one short -> reject.
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
KpoolTailManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
@@ -406,15 +404,15 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
@pytest.mark.parametrize("record_for_zeroing", [True, False])
def test_external_kpool_tail_zeroing(record_for_zeroing):
"""A transferred ring at index zero skips zeroing even with a local hit."""
spec = KpoolTailSpec(
"""The kpool tail ring is never zeroed, transferred or not: a pool only
reads ring slots its own request wrote."""
spec = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
sliding_window=4,
)
manager = KpoolTailManager(
manager = CircularBufferManager(
spec,
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
enable_caching=True,
@@ -428,9 +426,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
)
blocks = manager.req_to_blocks["request"]
assert len(blocks) == 1
assert manager.take_new_block_ids() == (
[blocks[0].block_id] if record_for_zeroing else []
)
assert manager.take_new_block_ids() == []
assert manager.allocate_new_blocks("request", 20, 20) == []
assert manager.take_new_block_ids() == []
+1 -1
View File
@@ -23,5 +23,5 @@ def _mock_cpu_backend_block_sizes(monkeypatch):
monkeypatch.setattr(
CPUAttentionBackend,
"get_supported_kernel_block_sizes",
staticmethod(lambda: [MultipleOf(16)]),
staticmethod(lambda kv_cache_spec=None: [MultipleOf(16)]),
)
@@ -164,13 +164,12 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
fa_spec = kv_cache_config.kv_cache_groups[0].kv_cache_spec
gdn_spec = kv_cache_config.kv_cache_groups[1].kv_cache_spec
fa_raw = torch.empty(num_blocks * fa_spec.page_size_bytes, dtype=torch.int8)
gdn_raw = torch.empty(num_blocks * gdn_spec.page_size_bytes, dtype=torch.int8)
gdn_block_stride = gdn_spec.page_size_bytes + 64
gdn_raw = torch.empty(num_blocks, gdn_block_stride, dtype=torch.int8)
(fa_cache,) = dense_kv_cache_views(
fa_raw, fa_spec, num_blocks, 1, KVCacheLayout.LBHNC
)
(gdn_cache,) = dense_kv_cache_views(
gdn_raw, gdn_spec, num_blocks, 1, KVCacheLayout.LBHNC
)
gdn_cache = gdn_raw[:, : gdn_spec.page_size_bytes]
worker.register_kv_caches(
{
@@ -184,7 +183,7 @@ def test_register_kv_caches_emits_fa_and_gdn_regions(monkeypatch):
"model.layers.0.self_attn",
"model.layers.1.linear_attn",
]
assert worker.block_len_per_layer == [
assert worker.kv_block_len_per_layer == [
fa_spec.page_size_bytes,
gdn_spec.page_size_bytes,
]
@@ -31,7 +31,6 @@ from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
@@ -711,13 +710,12 @@ def test_worker_setup_tolerates_finer_scratch_group():
mamba_cache_mode="align",
)
# Scratch block 4 is not divisible by the hash unit 8 below.
scratch = KpoolTailSpec(
scratch = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=64,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
)
cfg = KVCacheConfig(
num_blocks=4,
@@ -641,164 +641,43 @@ def _make_mla_hybrid_worker(local_block_size, kernel_block_size, num_logical_blo
@pytest.mark.cpu_test
@pytest.mark.parametrize("logical_block_size", [1152, 640])
@pytest.mark.parametrize("tail_first", [False, True])
def test_register_compressed_indexer_uses_virtual_transfer_pages(
logical_block_size, tail_first
):
"""Compressed indexer rows must split into contiguous NIXL transfer pages."""
from unittest.mock import MagicMock
def test_nixl_keeps_packed_sparse_mla_manager_block():
"""Like the worker, NIXL keeps packed sparse MLA + indexer manager blocks."""
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.v1.attention.backends.mla.flashmla_sparse import FlashMLASparseBackend
from vllm.v1.attention.backends.mla.indexer import Glm5NextIndexerBackend
from vllm.v1.kv_cache_interface import UniformTypeKVCacheSpecs
from vllm.config import set_current_vllm_config
from vllm.distributed.kv_transfer.kv_connector.v1.nixl import (
base_worker as bw,
specs = {
"mla": MLAAttentionSpec(
1152, num_kv_heads=1, head_size=512, dtype=torch.bfloat16
),
"indexer": MLAAttentionSpec(
1152, num_kv_heads=1, head_size=128, dtype=torch.uint8, tokens_per_state=4
),
}
backends = {"mla": FlashMLASparseBackend, "indexer": Glm5NextIndexerBackend}
layers = {
name: MagicMock(spec=AttentionLayerBase, **{"get_attn_backend.return_value": b})
for name, b in backends.items()
}
worker = object.__new__(bw.NixlBaseConnectorWorker)
worker.block_size, worker.num_blocks, worker.attn_backends = 1152, 2, []
worker._physical_blocks_per_logical_kv_block = 1
worker.vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
get_resolved_kv_cache_layout=lambda: KVCacheLayout.BLHNC
),
compilation_config=SimpleNamespace(static_forward_context=layers),
)
from vllm.distributed.kv_transfer.kv_connector.v1.nixl.worker import (
NixlConnectorWorker,
)
from vllm.v1.kv_cache_interface import (
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
KVCacheTensor,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
create_kv_cache_views,
worker.kv_cache_config = KVCacheConfig(
2, [], [KVCacheGroupSpec(list(specs), UniformTypeKVCacheSpecs(1152, specs))]
)
num_logical_blocks = 3
transfer_block_size = 64
kernel_block_size = 128
tokens_per_state = 4
state_content_bytes = 132
worker._sync_block_size_with_kernel()
indexer_spec = MLAAttentionSpec(
block_size=logical_block_size,
num_kv_heads=1,
head_size=128,
head_size_v=0,
dtype=torch.uint8,
state_content_bytes=state_content_bytes,
tokens_per_state=tokens_per_state,
)
indexer_page_size = indexer_spec.page_size_bytes
tail_spec = KpoolTailSpec(
block_size=tokens_per_state,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
page_size_padded=indexer_page_size,
sliding_window=tokens_per_state,
)
allocation_size = num_logical_blocks * indexer_page_size
indexer_tensor = KVCacheTensor(
size=allocation_size,
layers=["indexer"],
layer_stride=allocation_size,
block_stride=indexer_page_size,
)
tail_tensor = KVCacheTensor(
size=allocation_size,
layers=["tail"],
layer_stride=allocation_size,
block_stride=indexer_page_size,
)
kv_cache_config = KVCacheConfig(
num_blocks=num_logical_blocks,
kv_cache_tensors=[indexer_tensor, tail_tensor],
kv_cache_groups=[
KVCacheGroupSpec(
["indexer"],
UniformTypeKVCacheSpecs(
block_size=logical_block_size,
kv_cache_specs={"indexer": indexer_spec},
),
),
KVCacheGroupSpec(
["tail"],
UniformTypeKVCacheSpecs(
block_size=tokens_per_state,
kv_cache_specs={"tail": tail_spec},
),
),
],
)
raw = torch.zeros(allocation_size, dtype=torch.int8)
(indexer_cache,) = create_kv_cache_views(
raw,
indexer_spec,
num_logical_blocks,
KVCacheLayout.LBHNC,
indexer_tensor,
kernel_block_size=kernel_block_size,
)
(tail_cache,) = create_kv_cache_views(
raw,
tail_spec,
num_logical_blocks,
KVCacheLayout.LBHNC,
tail_tensor,
)
assert indexer_cache.data_ptr() == tail_cache.data_ptr() == raw.data_ptr()
vllm_config = create_vllm_config(block_size=logical_block_size)
vllm_config.cache_config.kv_cache_layout = "LBHNC"
vllm_config.kv_transfer_config.kv_buffer_device = "cuda"
fake_backend = MagicMock()
fake_backend.get_supported_kernel_block_sizes.return_value = [transfer_block_size]
fake_backend.get_name.return_value = "DEEPSEEK_V32_INDEXER"
fake_backend.full_cls_name.return_value = "fake.DEEPSEEK_V32_INDEXER"
fake_platform = MagicMock()
fake_platform.device_type = "cuda"
fake_platform.get_nixl_memory_type.return_value = "VRAM"
caches = [("indexer", indexer_cache), ("tail", tail_cache)]
if tail_first:
caches.reverse()
with (
patch.object(bw, "NixlWrapper", _RecordingNixl),
patch.object(bw, "get_tensor_model_parallel_rank", return_value=0),
patch.object(bw, "get_tensor_model_parallel_world_size", return_value=1),
patch.object(bw, "get_current_attn_backends", return_value=[fake_backend]),
patch.object(bw, "current_platform", fake_platform),
set_current_vllm_config(vllm_config),
):
worker = NixlConnectorWorker(vllm_config, "local-engine", kv_cache_config)
worker.use_mla = True
worker.register_kv_caches(dict(caches))
transfer_page_size = transfer_block_size // tokens_per_state * state_content_bytes
num_transfer_blocks = num_logical_blocks * (
logical_block_size // transfer_block_size
)
expected_descs = np.asarray(
[
[
raw.data_ptr() + block_idx * transfer_page_size,
transfer_page_size,
0,
]
for block_idx in range(num_transfer_blocks)
],
dtype=np.uint64,
)
assert worker.block_size == transfer_block_size
assert worker.num_regions == 1
assert worker.block_len_per_layer == [transfer_page_size]
assert worker.block_stride_per_layer == [transfer_page_size]
assert worker._region_is_mla == [True]
assert worker.kv_caches_base_addr[worker.engine_id][0] == [raw.data_ptr()]
assert worker._registered_descs[0] == [(raw.data_ptr(), raw.nbytes, 0, "")]
np.testing.assert_array_equal(worker.src_blocks_data, expected_descs)
assert expected_descs[-1, 0] + expected_descs[-1, 1] == (
raw.data_ptr() + raw.nbytes
)
assert worker.block_size == 1152
assert worker._physical_blocks_per_logical_kv_block == 1
def _make_remote_meta(
+1 -3
View File
@@ -47,7 +47,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheTensor,
@@ -145,13 +144,12 @@ def _make_scratch_kv_cache_config(
register_all_kvcache_specs(vllm_config=None)
fa_config = _make_kv_cache_config(num_blocks, num_groups=1)
scratch_layers = ["layer_scratch"]
scratch_spec = KpoolTailSpec(
scratch_spec = CircularBufferSpec(
block_size=scratch_block_size,
num_kv_heads=2,
head_size=HEAD_SIZE,
head_size_v=0,
dtype=DTYPE,
sliding_window=scratch_block_size,
)
assert not scratch_spec.prefix_cacheable
scratch_bytes = scratch_spec.page_size_bytes * num_blocks
@@ -102,7 +102,7 @@ def _run_prepare(
block_tables=SimpleNamespace(
slot_mappings=query_slot_mapping.unsqueeze(0),
input_block_tables=[block_table],
kernel_block_sizes=[4],
block_sizes=[4],
cp_rank=cp_rank,
cp_size=cp_size,
cp_interleave=cp_interleave,
+6
View File
@@ -963,6 +963,9 @@ def test_propose(method, attn_backend, num_speculative_tokens, monkeypatch):
# Mock draft_attn_groups for attention metadata building.
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.build_metadata_for_drafting = (
attn_metadata_builder.build_for_drafting
)
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
proposer.draft_attn_groups = [mock_attn_group]
@@ -1066,6 +1069,9 @@ def test_propose_stores_probabilistic_draft_probs(attn_backend, monkeypatch):
)
mock_attn_group = mock.MagicMock()
mock_attn_group.get_metadata_builder.return_value = attn_metadata_builder
mock_attn_group.build_metadata_for_drafting = (
attn_metadata_builder.build_for_drafting
)
mock_attn_group.layer_names = list(proposer._draft_attn_layer_names)
mock_attn_group.kv_cache_spec = attn_metadata_builder.kv_cache_spec
proposer.draft_attn_groups = [mock_attn_group]
+10 -14
View File
@@ -2,10 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for SpecDecodeBaseProposer.initialize_attn_backend.
Block tables are stored at kernel-block granularity, so the proposer's
``block_size`` (used for slot-mapping math) must be the kernel block size,
not the KV cache manager's block size — the two differ when manager blocks
are split for the attention kernel. The value must also be deterministic:
Block tables are stored at manager-block granularity, so the proposer's
``block_size`` (used for slot-mapping math) must be the KV cache manager's
block size. Attention groups map to kernel blocks for their metadata builders.
The value must also be deterministic:
``_draft_attn_layer_names`` is a set, whose iteration order varies across
processes, so anything derived from iteration order must not leak into
``block_size``.
@@ -68,9 +68,9 @@ def _make_kv_cache_config(layer_names: set[str]) -> KVCacheConfig:
return SimpleNamespace(kv_cache_groups=[group]) # type: ignore[return-value]
def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
"""The proposer's slot-mapping math runs against the kernel-granularity
block table, so block_size must come from kernel_block_sizes."""
def test_block_size_uses_manager_block_size(monkeypatch: pytest.MonkeyPatch):
"""The proposer's slot-mapping math runs against the manager-granularity
block table; attention groups map kernel blocks for metadata builders."""
layer_names = {"draft.0.self_attn.attn"}
proposer = _make_proposer(monkeypatch, layer_names)
@@ -79,12 +79,8 @@ def test_block_size_uses_kernel_block_size(monkeypatch: pytest.MonkeyPatch):
kernel_block_sizes=[KERNEL_BLOCK_SIZE],
)
assert proposer.block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size != SCHEDULER_BLOCK_SIZE
# The metadata builder keeps receiving the kernel block size as well.
group = proposer.draft_attn_groups[0]
assert isinstance(group, _FakeAttentionGroup)
assert group.kernel_block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
assert proposer.draft_attn_groups[0].kernel_block_size == KERNEL_BLOCK_SIZE
def test_block_size_falls_back_to_kv_cache_spec(monkeypatch: pytest.MonkeyPatch):
@@ -112,4 +108,4 @@ def test_draft_layer_iteration_is_deterministic(monkeypatch: pytest.MonkeyPatch)
)
assert len(proposer.draft_attn_groups) == 1
assert proposer.draft_attn_groups[0].layer_names == expected_order
assert proposer.block_size == KERNEL_BLOCK_SIZE
assert proposer.block_size == SCHEDULER_BLOCK_SIZE
@@ -38,7 +38,6 @@ def mock_model_runner_with_input_batch():
device="cpu",
vocab_size=32000,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
logitsprocs=None,
is_pooling_model=False,
+102 -10
View File
@@ -15,7 +15,11 @@ import pytest
import torch
import vllm.v1.hisparse.binding as attn_utils_module
from tests.v1.attention.utils import dense_kv_cache_views
from tests.v1.attention.utils import (
BatchSpec,
create_common_attn_metadata,
dense_kv_cache_views,
)
from vllm.config.compilation import CompilationConfig, CUDAGraphMode
from vllm.v1.attention.backend import AttentionBackend, AttentionCGSupport, MultipleOf
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
@@ -42,6 +46,8 @@ from vllm.v1.worker.utils import (
AttentionGroup,
allocate_kv_cache,
copy_kv_cache_blocks_inplace,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
)
@@ -79,7 +85,7 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
backend = SimpleNamespace(
get_name=lambda name=name: name,
customize_spec=AttentionBackend.customize_spec,
get_supported_kernel_block_sizes=lambda sizes=sizes: sizes,
get_supported_kernel_block_sizes=lambda _=None, sizes=sizes: sizes,
)
layers[name] = SimpleNamespace(
get_kv_cache_spec=lambda _, spec=specs[name]: spec,
@@ -99,7 +105,8 @@ def test_get_kv_cache_spec_resolves_hisparse_block_size(
resolved = attn_utils.get_kv_cache_spec(config)
assert resolved["main"].block_size == resolved["indexer"].block_size == expected
assert resolved["dense"] is specs["dense"]
dense_backend = layers["dense"].get_attn_backend()
assert resolved["dense"] == customize_attention_spec(dense_backend, specs["dense"])
assert all(spec.block_size == block_size for spec in specs.values())
@@ -495,20 +502,17 @@ def test_reshape_padded_kv_cache_strides_by_padded_page():
@pytest.mark.parametrize(
(
"kernel_block_sizes",
"storage_block_size",
"expected_num_blocks",
"expected_num_states",
),
[
(None, None, 4, 64),
([256], None, 4, 64),
([64], None, 16, 16),
([64], 256, 4, 64),
(None, 4, 64),
([256], 4, 64),
([64], 16, 16),
],
)
def test_allocate_compressed_mla_cache(
kernel_block_sizes: list[int] | None,
storage_block_size: int | None,
expected_num_blocks: int,
expected_num_states: int,
):
@@ -518,7 +522,6 @@ def test_allocate_compressed_mla_cache(
head_size=128,
dtype=torch.bfloat16,
tokens_per_state=4,
storage_block_size=storage_block_size,
)
num_pages = 4
config = KVCacheConfig(
@@ -744,3 +747,92 @@ def test_allocate_hisparse_kv_caches_host_pool_and_view_less_specs():
backing.untyped_storage().data_ptr()
== caches["indexer"].untyped_storage().data_ptr()
)
class _TableBuilder:
requires_block_table_width = False
supports_update_block_table = False
def __init__(self, kv_cache_spec, layer_names, vllm_config, device):
self.kv_cache_spec = kv_cache_spec
def build(self, common_prefix_len, common_attn_metadata, fast_build=False):
return common_attn_metadata.block_table_tensor
_VLLM_CONFIG = SimpleNamespace(
model_config=SimpleNamespace(max_model_len=8192),
scheduler_config=SimpleNamespace(max_num_seqs=4),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
)
def _packed_group(tokens_per_state, row_bytes, supported_sizes):
spec = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=128,
dtype=torch.uint8,
state_content_bytes=row_bytes,
tokens_per_state=tokens_per_state,
)
backend = SimpleNamespace(
get_builder_cls=lambda: _TableBuilder,
get_supported_kernel_block_sizes=lambda _=None: supported_sizes,
get_name=lambda: "FAKE",
customize_spec=lambda spec: spec,
)
return AttentionGroup(backend, ["layer.0"], spec, 0)
def test_packed_compressed_group_gets_kernel_block_table():
"""A packed indexer cache whose kernel needs 64-state pages gets a table and
a bound view in kernel blocks that address the same packed-block rows."""
row, stride_rows = 132, 5 * 64
group = _packed_group(4, row, [128, 256])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
assert group.get_metadata_builder().kv_cache_spec.block_size == 256
backing = torch.arange(3 * stride_rows).repeat_interleave(row)
manager = backing.view(3, 1, stride_rows, row)[:, :, :256]
pages = map_kv_caches_to_kernel_blocks({"layer.0": manager}, [group])["layer.0"]
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
table = group.build_metadata(metadata)
assert table.tolist() == [[0, 1, 2, 3, 10, 11, 12, 13]]
assert torch.equal(pages[table[0, 4:], 0].flatten(0, 1), manager[2, 0])
def test_split_group_maps_manager_block_table():
"""Groups split in a dense layout keep manager block tables; builders get
b * blocks_per_kv_block + j against the view allocated in kernel blocks."""
group = _packed_group(4, 132, [128, 256])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=256)
kernel_view = torch.zeros(12, 1, 64, 132)
bound = map_kv_caches_to_kernel_blocks({"layer.0": kernel_view}, [group])
assert bound["layer.0"] is kernel_view
metadata = create_common_attn_metadata(BatchSpec([2048], [1]), 1024, "cpu")
metadata.block_table_tensor = torch.tensor([[0, 2]], dtype=torch.int32)
table = group.build_metadata(metadata)
assert table.tolist() == [[0, 1, 2, 3, 8, 9, 10, 11]]
@pytest.mark.parametrize(
("block_size", "page_states"), [(1024, 64), (384, 32), (256, 0)]
)
def test_split_blocks_align_to_kernel_blocks(block_size, page_states):
"""Backends taking only fixed kernel blocks align packed blocks to the kernel
block the final block size is split into, and not at all when unsplit."""
group = _packed_group(4, 132, [128, 256])
spec = customize_attention_spec(group.backend, group.kv_cache_spec)
spec = spec.copy_with_new_block_size(block_size)
assert spec.get_block_stride_alignment() == max(page_states * 132, 1)
def test_packed_token_cache_cannot_be_split():
group = _packed_group(1, 1152, [64])
group.create_metadata_builders(_VLLM_CONFIG, "cpu", kernel_block_size=1024)
packed = torch.zeros(3, 1, 2048, 1152, dtype=torch.uint8)[:, :, :1024]
with pytest.raises(ValueError, match="cannot be split"):
map_kv_caches_to_kernel_blocks({"layer.0": packed}, [group])
+11 -20
View File
@@ -22,7 +22,6 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
max_num_batched_tokens=64,
max_num_blocks_per_group=[8, 8, 8],
device=device,
kernel_block_sizes=[16, 16, 8],
)
def fail_if_apply_write_called():
@@ -48,33 +47,31 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
block_tables.block_tables[0].gpu[0, :2],
torch.tensor([1, 2], dtype=torch.int32, device=device),
)
# Group 1 has blocks_per_kv_block == 2, so each KV block expands to two
# kernel block IDs.
assert torch.equal(
block_tables.block_tables[1].gpu[0, :4],
torch.tensor([20, 21, 22, 23], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[0, :2],
torch.tensor([10, 11], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[0].gpu[1, :1],
torch.tensor([3], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[1, :2],
torch.tensor([24, 25], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[1, :1],
torch.tensor([12], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[1, :2],
torch.tensor([5, 6], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 2
assert block_tables.num_blocks.np[1, 0] == 4
assert block_tables.num_blocks.np[1, 0] == 2
assert block_tables.num_blocks.np[2, 0] == 0
assert block_tables.num_blocks.np[0, 1] == 1
assert block_tables.num_blocks.np[1, 1] == 2
assert block_tables.num_blocks.np[1, 1] == 1
assert block_tables.num_blocks.np[2, 1] == 2
assert torch.equal(
block_tables.num_blocks.gpu[:, :2],
torch.tensor([[2, 1], [4, 2], [0, 2]], dtype=torch.int32, device=device),
torch.tensor([[2, 1], [2, 1], [0, 2]], dtype=torch.int32, device=device),
)
for block_table in block_tables.block_tables:
@@ -96,15 +93,15 @@ def test_block_tables_apply_staged_writes_fuses_kv_groups(monkeypatch):
torch.tensor([1, 2, 7], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[1].gpu[0, :6],
torch.tensor([20, 21, 22, 23, 26, 27], dtype=torch.int32, device=device),
block_tables.block_tables[1].gpu[0, :3],
torch.tensor([10, 11, 13], dtype=torch.int32, device=device),
)
assert torch.equal(
block_tables.block_tables[2].gpu[0, :1],
torch.tensor([8], dtype=torch.int32, device=device),
)
assert block_tables.num_blocks.np[0, 0] == 3
assert block_tables.num_blocks.np[1, 0] == 6
assert block_tables.num_blocks.np[1, 0] == 3
assert block_tables.num_blocks.np[2, 0] == 1
@@ -116,7 +113,6 @@ def test_block_tables_apply_staged_writes_single_group():
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[16],
)
block_tables.append_block_ids(
@@ -141,7 +137,6 @@ def test_block_tables_skip_custom_slot_mapping_groups():
max_num_batched_tokens=4,
max_num_blocks_per_group=[1, 1],
device=device,
kernel_block_sizes=[8, 262144],
slot_mapping_enabled=[False, True],
)
block_tables.append_block_ids(
@@ -170,7 +165,7 @@ def test_block_tables_skip_custom_slot_mapping_groups():
@pytest.mark.parametrize("cp_rank", range(4))
def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
def test_dcp_slot_mapping_only_shards_sharded_groups(cp_rank: int):
"""Only sharded groups use DCP interleave in logical-block coordinates."""
device = torch.device("cuda")
block_tables = BlockTables(
@@ -179,7 +174,6 @@ def test_dcp_slot_mapping_with_smaller_kernel_blocks(cp_rank: int):
max_num_batched_tokens=1024,
max_num_blocks_per_group=[2, 8],
device=device,
kernel_block_sizes=[64, 64],
dcp_sharded=[True, False],
cp_size=4,
cp_rank=cp_rank,
@@ -230,7 +224,6 @@ def test_v1_block_table_move_row_clears_vacated_row():
max_num_batched_tokens=64,
pin_memory=False,
device=torch.device("cuda"),
kernel_block_size=16,
cp_kv_cache_interleave_size=1,
)
block_table.add_row([7, 8, 9], row_idx=0)
@@ -260,7 +253,6 @@ def test_get_dummy_block_tables_returns_zeroed_rows():
max_num_batched_tokens=64,
max_num_blocks_per_group=[8],
device=device,
kernel_block_sizes=[16],
)
# Simulate a real step: stage a request's blocks and gather them into
# the persistent input block tables.
@@ -292,7 +284,6 @@ def test_dummy_request_slot_mapping_is_pad():
max_num_batched_tokens=16,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[4],
)
block_tables.append_block_ids(req_index=0, new_block_ids=([7, 8],), overwrite=True)
block_tables.apply_staged_writes()
-9
View File
@@ -233,7 +233,6 @@ def test_sampling_metadata_in_input_batch(device: str, batch_size: int):
device=torch.device(device),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
reqs: list[CachedRequestState] = []
@@ -327,7 +326,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
device=torch.device(device),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
ref_input_batch: InputBatch = InputBatch(
@@ -337,7 +335,6 @@ def test_swap_states_in_input_batch(device: str, batch_size: int, swap_list: lis
device=torch.device(device),
vocab_size=1024,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
@@ -388,7 +385,6 @@ def test_condense_clears_stale_allowed_token_ids_mask(device: str):
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
@@ -446,7 +442,6 @@ def test_swap_states_preserves_allowed_token_ids_mask(device: str):
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[1],
kernel_block_sizes=[1],
max_num_blocks_per_req=[1024],
)
@@ -516,7 +511,6 @@ def test_pooling_prompt_lens_not_aliased(device: str):
device=torch.device(device),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
is_pooling_model=True,
)
@@ -552,7 +546,6 @@ def test_placeholder_spec_token_ids_written_verbatim():
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[1],
)
req = CachedRequestState(
@@ -599,7 +592,6 @@ def test_pooling_metadata_token_id_buffers(
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
is_pooling_model=True,
)
@@ -630,7 +622,6 @@ def _make_input_batch(is_pooling_model: bool = False) -> InputBatch:
device=torch.device("cpu"),
vocab_size=VOCAB_SIZE,
block_sizes=[16],
kernel_block_sizes=[16],
max_num_blocks_per_req=[64],
is_pooling_model=is_pooling_model,
)
@@ -5,7 +5,7 @@
The kpool tail cache holds one circular block per request, so its block table
row is orders of magnitude shorter than the sequence it serves.
`_compute_slot_mappings_kernel` indexes block tables by
`position // kernel_block_size` with an effectively unmasked load, so
`position // block_size` with an effectively unmasked load, so
this leads to a read of neighboring requests' rows (and for the later requests, outside
the buffer entirely).
"""
@@ -15,7 +15,7 @@ import torch
from vllm.platforms import current_platform
from vllm.v1.attention.backends.utils import PAD_SLOT_ID
from vllm.v1.kv_cache_interface import KpoolTailSpec
from vllm.v1.kv_cache_interface import CircularBufferSpec
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.gpu.block_table import BlockTables
@@ -32,13 +32,12 @@ POISON_BLOCK_ID = 1 << 20
def make_tail_spec():
return KpoolTailSpec(
return CircularBufferSpec(
block_size=KPOOL,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=KPOOL,
)
@@ -67,7 +66,6 @@ def test_kpool_tail_group_never_position_indexes_its_block_table():
max_num_batched_tokens=len(positions),
max_num_blocks_per_group=[mla_width, tail_width],
device=device,
kernel_block_sizes=[MLA_BLOCK_SIZE, spec.block_size],
slot_mapping_enabled=[True, spec.uses_slot_mapping],
)
mla_blocks = list(range(1, mla_width + 1))
+4 -104
View File
@@ -111,9 +111,6 @@ def initialize_kv_cache(runner: GPUModelRunner):
device=runner.device,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
],
max_num_blocks_per_req=[NUM_BLOCKS],
)
runner.initialize_attn_backend(kv_cache_config)
@@ -303,7 +300,7 @@ def _make_mock_backend_for_kernel_block_size(
return "MOCK"
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return supported_sizes
return _MockBackend()
@@ -441,7 +438,7 @@ def _mock_backend(supported: list, *, exact: bool = False):
return "MOCK_EXACT" if exact else "MOCK"
@staticmethod
def get_supported_kernel_block_sizes():
def get_supported_kernel_block_sizes(kv_cache_spec=None):
return list(supported)
if exact:
@@ -1625,7 +1622,6 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
runner.device = torch.device("cpu")
runner.is_pooling_model = False
runner._init_block_sizes = [16]
runner._init_kernel_block_sizes = [16]
runner._init_max_num_blocks = [4]
runner._init_slot_mapping_modes = [
gpu_model_runner_module.SlotMappingMode.TOKEN_TO_KV_SLOT
@@ -1653,7 +1649,7 @@ def test_input_batch_reinitialized_after_late_interleave_adjustment(monkeypatch)
lambda _: gpu_model_runner_module.KVCacheSpecKind.FULL_ATTENTION,
)
runner.may_reinitialize_input_batch(kv_cache_config, [16])
runner.may_reinitialize_input_batch(kv_cache_config)
assert input_batch_cls.call_count == 1
assert input_batch_cls.call_args.kwargs["cp_kv_cache_interleave_size"] == 16
@@ -1681,68 +1677,10 @@ def test_v2_runner_snapshots_late_interleave_adjustment(monkeypatch):
assert runner.cp_interleave == 16
def test_hybrid_block_table_initialization():
"""Test hybrid block table with different kernel and kvcache_manager block
sizes."""
from vllm.v1.worker.block_table import BlockTable
# Test configuration: kvcache_manager block size = 32,
# kernel block size = 16
block_size = 32
kernel_block_sizes = [16]
max_num_reqs = 10
max_num_blocks_per_req = 20
max_num_batched_tokens = 512
cp_kv_cache_interleave_size = 8
block_table = BlockTable(
block_size=block_size,
max_num_reqs=max_num_reqs,
max_num_blocks_per_req=max_num_blocks_per_req,
max_num_batched_tokens=max_num_batched_tokens,
pin_memory=False,
device=torch.device(DEVICE_TYPE),
kernel_block_size=kernel_block_sizes[0],
cp_kv_cache_interleave_size=cp_kv_cache_interleave_size,
)
# Verify hybrid block configuration
assert block_table.use_hybrid_blocks is True
assert block_table.block_size == kernel_block_sizes[0]
assert block_table.blocks_per_kv_block == (
block_size // kernel_block_sizes[0]
) # Changed to use first element
# Test block table conversion logic
# One kvcache_manager block should map to multiple kernel blocks
kvcache_manager_blocks = [0, 1, 2]
# Verify that kvcache_manager blocks can be converted to kernel blocks
# and that block table operations work correctly.
req_index = 0
block_table.append_row(kvcache_manager_blocks, req_index)
# Get expected kernel blocks from the implementation for verification.
expected_kernel_blocks = block_table.map_to_kernel_blocks(
np.array(kvcache_manager_blocks),
block_table.blocks_per_kv_block,
block_table._kernel_block_arange,
)
# Verify block table state
assert block_table.num_blocks_per_row[req_index] == len(expected_kernel_blocks)
assert np.array_equal(
block_table.block_table.np[req_index, : len(expected_kernel_blocks)],
expected_kernel_blocks,
)
def test_get_block_table_width_aligns_to_128_tokens():
assert get_block_table_width(1875, 64) == 1876
def test_get_block_table_width_splits_virtual_blocks():
assert get_block_table_width(235, 256, 64) == 940
def test_mamba_state_table_width_is_not_aligned():
block_tables = MultiGroupBlockTable(
max_num_reqs=1,
@@ -1750,7 +1688,6 @@ def test_mamba_state_table_width_is_not_aligned():
pin_memory=False,
device=torch.device("cpu"),
block_sizes=[39664],
kernel_block_sizes=[39664],
max_num_blocks=[1],
slot_mapping_modes=[SlotMappingMode.NONE],
)
@@ -1758,42 +1695,6 @@ def test_mamba_state_table_width_is_not_aligned():
assert block_tables[0].max_num_blocks_per_req == 1
def test_input_batch_with_kernel_block_sizes():
"""Test InputBatch initialization with kernel_block_sizes parameter."""
max_num_reqs = 10
max_model_len = 512
max_num_batched_tokens = 512
device = torch.device(DEVICE_TYPE)
vocab_size = 50272
# Test with different kernel block sizes
block_sizes = [32, 64]
kernel_block_sizes = [16, 32]
input_batch = InputBatch(
max_num_reqs=max_num_reqs,
max_model_len=max_model_len,
max_num_batched_tokens=max_num_batched_tokens,
device=device,
vocab_size=vocab_size,
block_sizes=block_sizes,
kernel_block_sizes=kernel_block_sizes,
max_num_blocks_per_req=[16, 8],
)
# Verify that block tables were created with kernel block sizes
assert len(input_batch.block_table.block_tables) == len(block_sizes)
for i, (kv_size, kernel_size) in enumerate(zip(block_sizes, kernel_block_sizes)):
block_table = input_batch.block_table.block_tables[i]
if kv_size != kernel_size:
assert block_table.use_hybrid_blocks is True
assert block_table.block_size == kernel_size
else:
assert block_table.use_hybrid_blocks is False
assert block_table.block_size == kernel_size
def test_hybrid_cache_integration(default_vllm_config, dist_init):
"""Test hybrid cache architecture integration with GPUModelRunner."""
# Create a new model runner with hybrid cache configuration
@@ -1843,9 +1744,8 @@ def test_hybrid_cache_integration(default_vllm_config, dist_init):
device=runner.device,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[16],
max_num_blocks_per_req=[NUM_BLOCKS],
) # Use kernel block size
)
runner.initialize_attn_backend(kv_cache_config)
@@ -119,9 +119,6 @@ def _initialize_kv_cache(runner: GPUModelRunner) -> None:
device=runner.device,
vocab_size=runner.model_config.get_vocab_size(),
block_sizes=[kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size],
kernel_block_sizes=[
kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
],
max_num_blocks_per_req=[NUM_BLOCKS],
)
runner.initialize_attn_backend(kv_cache_config)
@@ -237,7 +237,6 @@ def test_append_block_ids_rejects_write_past_row_capacity():
block_tables = BlockTables.__new__(BlockTables)
block_tables.num_kv_cache_groups = 1
block_tables.blocks_per_kv_block = [1]
block_tables.block_tables = [_BlockTable()]
block_tables.num_blocks = SimpleNamespace(
np=torch.tensor([[0, 3]], dtype=torch.int32)
+1 -2
View File
@@ -47,8 +47,7 @@ def _make_capture_manager(block_table: torch.Tensor):
block_tables = SimpleNamespace(
input_block_tables=(block_table,),
num_kv_cache_groups=1,
kernel_block_sizes=(2,),
blocks_per_kv_block=(1,),
block_sizes=(2,),
)
manager = PCPManager(
pcp_world_size=2,
@@ -20,6 +20,7 @@ from vllm.model_executor.models.mistral_large_3_eagle import (
from vllm.v1.attention.backends import flash_attn as flash_attn_module
from vllm.v1.attention.backends.flash_attn import FlashAttentionMetadata
from vllm.v1.attention.backends.utils import split_decodes_and_prefills
from vllm.v1.kv_cache_interface import MLAAttentionSpec
from vllm.v1.worker.gpu.cudagraph_utils import BatchExecutionDescriptor
from vllm.v1.worker.gpu.spec_decode import speculator as base_spec_module
from vllm.v1.worker.gpu.spec_decode.multi_module_mtp.speculator import (
@@ -30,6 +31,7 @@ from vllm.v1.worker.gpu.spec_decode.target_dependent_ar import speculator as spe
from vllm.v1.worker.gpu.spec_decode.target_dependent_ar.speculator import (
TargetDependentARSpeculator,
)
from vllm.v1.worker.utils import AttentionGroup
class _TestSpeculator(TargetDependentARSpeculator):
@@ -102,6 +104,7 @@ def test_pcp_draft_metadata_keeps_graph_padding_in_decode(cg_mode):
get_metadata_builder=lambda _: SimpleNamespace(
build=build, supports_update_block_table=False
),
build_metadata=lambda cm, ubatch_idx=0, **_: build(0, cm),
layer_names=["draft"],
)
]
@@ -427,6 +430,8 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
query_start_loc=torch.arange(3),
)
speculator.idx_mapping = torch.arange(2)
speculator.attn_groups = []
speculator.block_tables = SimpleNamespace(input_block_tables=[])
generate_draft = Mock()
speculator._generate_draft = generate_draft
run_fullgraph = Mock()
@@ -450,6 +455,52 @@ def test_multi_step_decode_replays_captured_graph_as_expected(
assert run_fullgraph.call_count == expected_graph_replays
def test_fused_full_graph_maps_split_group_block_tables():
"""The fused FULL replay skips build(), so split groups' tables are mapped first."""
spec = MLAAttentionSpec(
block_size=1024, num_kv_heads=1, head_size=128, dtype=torch.uint8
)
group = AttentionGroup(
SimpleNamespace(),
["layer.0"],
spec,
0,
kernel_block_size=256,
kernel_block_table=torch.zeros(1, 4, 32, dtype=torch.int32),
kernel_block_offsets=torch.arange(4, dtype=torch.int32),
)
speculator = object.__new__(_TestSpeculator)
speculator.input_buffers = SimpleNamespace(
positions=torch.arange(2), query_start_loc=torch.arange(3)
)
speculator.idx_mapping = torch.arange(2)
speculator.attn_groups = [[group]]
speculator.block_tables = SimpleNamespace(
input_block_tables=[torch.tensor([[1, 2], [0, 3]], dtype=torch.int32)]
)
replayed_tables = []
speculator.decode_cudagraph_manager = SimpleNamespace(
run_fullgraph=lambda _: replayed_tables.append(
group.kernel_block_table[0, :2, :8].tolist()
)
)
speculator._fused_multi_step_decode(
num_reqs=2,
skip_attn=False,
batch_desc=BatchExecutionDescriptor(
cg_mode=CUDAGraphMode.FULL, num_tokens=2, num_reqs=2
),
num_tokens_across_dp=None,
seq_lens_cpu_upper_bound=None,
num_speculative_steps=4,
)
assert replayed_tables == [
[[4, 5, 6, 7, 8, 9, 10, 11], [0, 1, 2, 3, 12, 13, 14, 15]]
]
def test_update_draft_decode_metadata_updates_fa3_scheduler_metadata(
monkeypatch,
):
+1 -17
View File
@@ -12,40 +12,24 @@ if not current_platform.is_cuda_alike():
from vllm.v1.worker.block_table import ComputeSlotMappingKernel
@pytest.mark.parametrize(
("kv_cache_block_size", "blocks_per_kv_block", "block_size", "block_size_rep"),
[
(256, 1, 256, 16),
(256, 4, 64, 16),
(64, 1, 64, 16),
(8, 1, 8, 2),
(4, 1, 4, 2),
],
)
@pytest.mark.parametrize("kv_cache_block_size", [256, 64, 8, 4])
def test_compute_slot_mapping_warmup_matches_runtime_specializations(
kv_cache_block_size: int,
blocks_per_kv_block: int,
block_size: int,
block_size_rep: int,
) -> None:
kernel = ComputeSlotMappingKernel()
kwargs = dict(
kv_cache_block_size=kv_cache_block_size,
blocks_per_kv_block=blocks_per_kv_block,
total_cp_world_size=1,
total_cp_rank=0,
cp_kv_cache_interleave_size=1,
block_table_stride=32768,
block_size=block_size,
)
expected = kernel.CompileKey(
kv_cache_block_size=kv_cache_block_size,
blocks_per_kv_block=blocks_per_kv_block,
total_cp_world_size=1,
total_cp_rank=0,
cp_kv_cache_interleave_size=1,
block_table_stride=16,
block_size=block_size_rep,
)
assert kernel.dispatch(**kwargs) == expected
@@ -112,6 +112,8 @@ def test_mrv1_kv_pool_only_wraps_backing_allocation(monkeypatch) -> None:
model_config=SimpleNamespace(hf_config=SimpleNamespace(model_type="test")),
compilation_config=SimpleNamespace(static_forward_context={}),
kv_caches=[],
attn_groups=[],
speculative_config=None,
)
result = gpu_model_runner.GPUModelRunner.initialize_kv_cache_tensors(
runner,
-1
View File
@@ -659,7 +659,6 @@ def test_v2_align_ctx_binds_request_slot_tables_for_state_copies():
max_num_batched_tokens=64,
max_num_blocks_per_group=[4],
device=device,
kernel_block_sizes=[cfg.block_size],
)
state = object.__new__(MambaHybridModelState)
+37
View File
@@ -443,6 +443,43 @@ def test_copy_cpu_kv_cache_logical_blocks_ignores_storage_padding():
assert (backing[9] == -1).all()
def test_bind_kv_cache_keeps_scheduler_block_views_for_runner():
"""Block copies index scheduler blocks, so the runner keeps the unmapped
views while a packed layer is bound with kernel blocks."""
class _Layer:
def bind_kv_cache(self, kv_cache):
self.kv_cache = kv_cache
num_blocks, row = 4, 64
storage = torch.zeros(num_blocks * row)
# Two layers packed into one block row: an 8x4 page, then a 4x2 page
# split into two 2-row kernel blocks.
mla = storage.as_strided((num_blocks, 8, 4), (row, 4, 1))
indexer = storage.as_strided((num_blocks, 1, 4, 2), (row, 8, 2, 1), 40)
# The indexer in 2-row kernel blocks: block b's kernel block j is b * 16 + j.
mapped = indexer.as_strided((50, 1, 2, 2), (4, 8, 2, 1))
kv_caches = {"layers.0.attn": mla, "layers.1.indexer": indexer}
ctx = {name: _Layer() for name in kv_caches}
runner_kv_caches: list[torch.Tensor] = []
bind_kv_cache(
kv_caches,
ctx,
runner_kv_caches,
layer_kv_caches={**kv_caches, "layers.1.indexer": mapped},
)
assert ctx["layers.1.indexer"].kv_cache is mapped
assert runner_kv_caches == [mla, indexer]
copies = [KVCacheBlockCopy(1, 3)]
with pytest.raises(AssertionError, match="not divisible"):
copy_kv_cache_blocks_inplace([mapped], num_blocks, copies)
storage[row : 2 * row] = 7
copy_kv_cache_blocks_inplace(runner_kv_caches, num_blocks, copies)
assert (mla[3] == 7).all() and (indexer[3] == 7).all()
@pytest.mark.parametrize("shared,rank", [(False, 0), (True, 0), (True, 1)])
@pytest.mark.parametrize("has_copies", [False, True])
def test_hisparse_host_copy_waits_for_writes_on_writer_only(