[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
@@ -26,8 +26,8 @@ from vllm.v1.core.kv_cache_utils import (
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
@@ -2308,12 +2308,11 @@ def test_opted_out_scratch_group_keeps_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=mamba_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=mamba_block_size,
),
),
],
@@ -2366,12 +2365,11 @@ def test_kpool_tail_supports_128_token_partial_hash_hits():
),
KVCacheGroupSpec(
["tail"],
KpoolTailSpec(
CircularBufferSpec(
block_size=kpool_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=kpool_block_size,
),
),
],
+32 -10
View File
@@ -88,7 +88,7 @@ def test_packed_alignment_preserves_hot_pages_and_generic_stride():
hot = HiSparseHotSpec(block_size=16, page_size=5120, blocks_per_request=2)
groups = [KVCacheGroupSpec(["attention"], spec), KVCacheGroupSpec(["hot"], hot)]
alignment = lcm(1152, hot.page_size_bytes)
stride = _pool_bytes_per_block(groups)
stride = _pool_bytes_per_block(_mock_vllm_config("BLHNC"), groups)
assert stride == alignment
layout = KVCacheLayout.BLHNC
tensors = _build_hisparse_kv_cache_tensors(groups, 3, 3 * stride, layout, stride)
@@ -254,7 +254,7 @@ class TestCSALinearGrouping:
def test_every_group_fits_one_packed_block(self):
config = _shared_layout_config()
groups = get_kv_cache_groups(config, _make_csa_linear_specs())
bytes_per_block = _get_kv_cache_bytes_per_block(groups)
bytes_per_block = _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC)
pages = _pages(groups)
for group in groups:
@@ -418,7 +418,7 @@ class TestCSALinearGrouping:
groups = _get_packed_kv_cache_groups(config, specs)
gdn = [g for g in groups if g.layer_names[0].startswith("gdn.")]
assert _get_kv_cache_bytes_per_block(groups) == sum(
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == sum(
specs[name].page_size_bytes for name in specs if name.startswith("wide.")
)
# That block holds every GDN state at once, so the repeat pattern alone
@@ -465,7 +465,7 @@ class TestSlidingWindowBucketCap:
pages = _pages(groups)
main = next(g for g in groups if "layers.2.attn" in g.layer_names)
main_bytes = sum(pages[n] for n in main.layer_names)
assert _get_kv_cache_bytes_per_block(groups) == main_bytes
assert _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC) == main_bytes
swa_groups = [g for g in groups if g.layer_names[0].endswith(".swa")]
per_group = main_bytes // pages["layers.0.swa"]
assert len(swa_groups) == -(-43 // per_group)
@@ -479,9 +479,9 @@ class TestSlidingWindowBucketCap:
class TestDensePacking:
def test_bytes_per_block_is_largest_group(self):
groups, g1, g2 = _mixed_page_groups()
assert _get_kv_cache_bytes_per_block(groups) == _expected_bytes_per_block(
groups
)
assert _get_kv_cache_bytes_per_block(
groups, KVCacheLayout.BLHNC
) == _expected_bytes_per_block(groups)
config = get_kv_cache_config_from_groups(
_mock_vllm_config("BLHNC"), groups, MEMORY
@@ -554,6 +554,20 @@ class TestDensePacking:
assert idx_tensor.offset == 2 * _mla(512).page_size_bytes
assert mla_tensor.block_stride == block_stride
def test_layer_outer_pads_only_unaligned_pages(self):
"""Layer-outer block strides are pages, so only an unaligned page pads."""
mla_page = _mla(512).page_size_bytes
idx = replace(_mla(128), block_stride_alignment=12288)
groups = [_uniform_group({"mla.0": _mla(512), "idx.0": idx})]
config = get_kv_cache_config_from_groups(
_mock_vllm_config("LBNHC"), groups, MEMORY
)
mla_tensor, idx_tensor = config.kv_cache_tensors
assert config.num_blocks == MEMORY // (mla_page + 12288)
assert (mla_tensor.block_stride, idx_tensor.block_stride) == (mla_page, 12288)
assert idx_tensor.offset == mla_page * config.num_blocks
_bind(config, "LBNHC")
def test_overlaid_groups_alias_and_stay_isolated(self):
groups, g1, g2 = _mixed_page_groups()
# Overlay models resolve to a block-outer layout at backend selection (the
@@ -562,7 +576,9 @@ class TestDensePacking:
_mock_vllm_config("BLNHC"), groups, MEMORY
)
assert config.num_blocks == MEMORY // _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(groups) == _expected_bytes_per_block(groups)
assert _pool_bytes_per_block(
_mock_vllm_config("BLNHC"), groups
) == _expected_bytes_per_block(groups)
views = _bind(config, "BLNHC")
assert set(views) == set(g1) | set(g2)
@@ -667,7 +683,10 @@ class TestCompressorRingGroup:
assert len(ring_groups) == 1
assert not ring_groups[0].kv_cache_spec.prefix_cacheable
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
assert resolve_kv_cache_block_sizes(kv_cache_config, config) == (128, 64)
manager = KVCacheManager(
@@ -716,7 +735,10 @@ class TestSWABoundedReplayGrouping:
for g in groups
) == [(False, 128)] * 4 + [(True, 0)]
kv_cache_config = get_kv_cache_config_from_groups(
config, groups, available_memory=64 * _get_kv_cache_bytes_per_block(groups)
config,
groups,
available_memory=64
* _get_kv_cache_bytes_per_block(groups, KVCacheLayout.BLHNC),
)
manager = KVCacheManager(
generate_scheduler_kv_cache_config([kv_cache_config]),
+57 -545
View File
@@ -10,7 +10,7 @@ from collections.abc import Callable
from dataclasses import replace
from pathlib import Path
from types import SimpleNamespace
from typing import Any, cast
from typing import Any
import pytest
import torch
@@ -19,7 +19,6 @@ import vllm.v1.core.kv_cache_utils as kv_cache_utils
import vllm.v1.hisparse.runtime as hisparse_runtime_module
from vllm.config import (
CacheConfig,
KVTransferConfig,
ModelConfig,
SchedulerConfig,
VllmConfig,
@@ -69,7 +68,6 @@ from vllm.v1.kv_cache_interface import (
HiddenStateCacheSpec,
HiSparseHotSpec,
HiSparseResidentSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheSpec,
@@ -1942,13 +1940,12 @@ def test_dcp_replicated_kpool_tail_keeps_block_interior_layout(monkeypatch):
config = VllmConfig(model_config=ModelConfig(max_model_len=1024))
config.parallel_config.decode_context_parallel_size = 4
config.cache_config.kv_cache_layout = None
tail = KpoolTailSpec(
tail = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=4,
dcp_sharded=False,
)
layout = resolve_kv_cache_layout(config, [["LBHNC"]], [new_mla_spec(), tail])
@@ -2906,564 +2903,77 @@ def test_generate_scheduler_kv_cache_config():
)
def _glm5_like_kv_cache_spec(
mamba_spec_factory=new_mamba_spec,
) -> tuple[dict[str, KVCacheSpec], list[str]]:
"""(mamba, mamba, mamba, MLA + indexer) * 11 plus a trailing mamba."""
def test_packed_groups_glm5_like_hybrid():
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=8192))
vllm_config.cache_config.kv_cache_layout = "BLHNC"
kpool, block_size = 4, 1024
kv_cache_spec: dict[str, KVCacheSpec] = {}
mamba_layers = []
for i in range(45):
if i % 4 == 3:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=16,
)
else:
name = f"layers.{i}.linear_attn"
kv_cache_spec[name] = mamba_spec_factory()
mamba_layers.append(name)
return kv_cache_spec, mamba_layers
def _glm5_like_kv_cache_spec_with_tail(
mamba_spec_factory=new_mamba_spec,
) -> dict[str, KVCacheSpec]:
"""Production kpool=4 proportions: (mamba*3, MLA + indexer + tail) * 11.
The tail's logical page (2 * kpool * 2*indexer_head_dim * bf16) must fit
inside the indexer page it parasitizes (block_size//kpool * 132 B), which
is why the fixture drops the tokens_per_state=16 shape used above.
"""
kv_cache_spec, _ = _glm5_like_kv_cache_spec(mamba_spec_factory)
for i in range(3, 45, 4):
kpool = 4
kv_cache_spec[f"layers.{i}.indexer"] = replace(
cast(MLAAttentionSpec, kv_cache_spec[f"layers.{i}.indexer"]),
tokens_per_state=kpool,
for i in range(11):
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
block_stride_alignment=32 * 576 * 2,
)
kv_cache_spec[f"layers.{i}.tail"] = KpoolTailSpec(
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=kpool,
block_stride_alignment=64 * 132,
)
kv_cache_spec[f"layers.{i}.tail"] = CircularBufferSpec(
block_size=kpool,
num_kv_heads=2,
head_size=128,
head_size_v=0,
dtype=torch.bfloat16,
sliding_window=kpool,
)
return kv_cache_spec
def _tensor_by_layer(kv_cache_config: KVCacheConfig) -> dict[str, KVCacheTensor]:
return {
layer_name: tensor
for tensor in kv_cache_config.kv_cache_tensors
for layer_name in tensor.layers
}
def _layer_offset(tensor: KVCacheTensor, layer_name: str) -> int:
return tensor.offset + tensor.layers.index(layer_name) * tensor.layer_stride
def test_get_kv_cache_config_balanced_mamba_hybrid():
"""Hybrid slot sharing: mamba layers co-own the MLA slot tensors."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
uniform_groups = [
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
]
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
# Round-robin into G = ceil(34 / 11) = 4 groups so every mamba layer
# gets an MLA slot.
assert len(groups) == 5
assert len(uniform_groups) == 1
assert len(uniform_groups[0].layer_names) == 22
assert uniform_groups[0].kv_cache_spec.get_max_layers_per_page_size() == 11
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
for k, name in enumerate(mamba_layers):
assert name in mamba_groups[k % 4].layer_names
# Mamba pages are padded up to the MLA page.
for group in mamba_groups:
assert group.kv_cache_spec.page_size_padded == mla_page
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
# Every block id is charged the full per-block sum.
attn_blocks = uniform_groups[0].kv_cache_spec.max_memory_usage_pages(vllm_config)
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group
assert (
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
== blocks_per_request * bytes_per_block
)
available_memory = bytes_per_block * 100 + 1
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, available_memory
)
assert kv_cache_config.num_blocks == 100
# Every logical layer has a view into one backing allocation. Mamba views
# alias their corresponding MLA slot by using the same byte offset.
assert len(kv_cache_config.kv_cache_tensors) == 56
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
tensors = _tensor_by_layer(kv_cache_config)
mla_layer_names = [f"layers.{i}.attn" for i in range(3, 45, 4)]
for i, mla_name in enumerate(mla_layer_names):
mla_tensor = tensors[mla_name]
assert mla_tensor.block_stride == mla_page
for group in mamba_groups:
if i < len(group.layer_names):
assert tensors[group.layer_names[i]].offset == mla_tensor.offset
for i in range(11):
assert tensors[f"layers.{4 * i + 3}.indexer"].block_stride == idx_page
total_allocated = next(iter({t.size for t in kv_cache_config.kv_cache_tensors}))
assert total_allocated == bytes_per_block * 100
assert 0 <= available_memory - total_allocated < bytes_per_block
assert get_max_concurrency_for_kv_cache_config(
vllm_config, kv_cache_config
) == pytest.approx(100 / blocks_per_request)
def test_get_kv_cache_config_kpool_tail_coowns_indexer_tensor():
"""The kpool tail parasitizes the indexer tensors instead of getting its
own: sibling idx/tail tensors paired by layer order, zero standalone tail
bytes, one shared block per request, and no prefix-caching leakage from
the kpool-sized scratch group."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec = _glm5_like_kv_cache_spec_with_tail()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
tail_logical_page = kv_cache_spec["layers.3.tail"].page_size_bytes
assert tail_logical_page < idx_page
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
tail_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
and all(
isinstance(spec, KpoolTailSpec)
for spec in group.kv_cache_spec.kv_cache_specs.values()
for i in range(34):
kv_cache_spec[f"layers.{i}.kda"] = MambaSpec(
shapes=((3, 6144), (16, 128, 128)),
dtypes=(torch.bfloat16, torch.float32),
block_size=block_size,
)
)
# The tail never prefix-caches; its page is padded up to the indexer page
# so the runner's strided view rides the indexer storage.
assert not tail_group.kv_cache_spec.prefix_cacheable
tail_inner = cast(
KpoolTailSpec, tail_group.kv_cache_spec.kv_cache_specs["layers.3.tail"]
)
assert tail_inner.page_size_padded == idx_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
# Tail layers get logical views but no additional storage: each view aliases
# its sibling indexer at the same offset in the shared allocation.
assert len(kv_cache_config.kv_cache_tensors) == 67
tensors = _tensor_by_layer(kv_cache_config)
for i in range(11):
idx_name = f"layers.{4 * i + 3}.indexer"
tail_name = f"layers.{4 * i + 3}.tail"
assert tensors[idx_name].offset == tensors[tail_name].offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
# The layout detector surfaces the sibling names in layer order for the
# accounting and connector paths.
layout = kv_cache_utils._glm5_next_tensor_layout(kv_cache_config.kv_cache_groups)
assert layout is not None
assert layout[3] == [f"layers.{4 * i + 3}.indexer" for i in range(11)]
assert layout[6] == [f"layers.{4 * i + 3}.tail" for i in range(11)]
# Accounting charges exactly one extra shared block per request for the
# tail, not a per-sequence allocation.
attn_group = next(
group
for group in groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
and group is not tail_group
)
attn_blocks = attn_group.kv_cache_spec.max_memory_usage_pages(vllm_config)
mamba_blocks_per_group = 1 + new_mamba_spec().num_speculative_blocks
blocks_per_request = attn_blocks + 4 * mamba_blocks_per_group + 1
assert (
kv_cache_utils._max_memory_usage_bytes_from_groups(vllm_config, groups)
== blocks_per_request * bytes_per_block
)
def test_glm5_kpool_tail_does_not_drag_hash_block_size():
"""The tail's kpool-sized scratch block (4 tokens) must not constrain the
prefix-cache hash granularity: participating groups alone decide it."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
def align_mamba():
return new_mamba_spec(mamba_cache_mode="align")
groups = kv_cache_utils.get_kv_cache_groups(
vllm_config, _glm5_like_kv_cache_spec_with_tail(align_mamba)
)
kv_cache_config = KVCacheConfig(
num_blocks=1,
kv_cache_tensors=[],
kv_cache_groups=groups,
)
hash_vllm_config = SimpleNamespace(
cache_config=SimpleNamespace(
block_size=16,
enable_prefix_caching=True,
prefix_match_unit=None,
),
parallel_config=SimpleNamespace(decode_context_parallel_size=1),
kv_transfer_config=object(),
)
# gcd(attn 1024, mamba 16) with the tail's 4 excluded; scheduler size is
# the lcm including the tail (1024 % 4 == 0, so it coincides).
assert kv_cache_utils.resolve_kv_cache_block_sizes(
kv_cache_config, hash_vllm_config
) == (1024, 16)
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible():
"""Reject GLM-5.3-Flash layouts whose Mamba page exceeds the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
# (512, 1024) fp32 state = 2 MiB per page > 1.125 MiB MLA page.
def big_mamba_spec():
return new_mamba_spec(shapes=((512, 1024),), dtypes=(torch.float32,))
kv_cache_spec, _ = _glm5_like_kv_cache_spec(big_mamba_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert big_mamba_spec().page_size_bytes > mla_page
with pytest.raises(ValueError, match="does not fit the MLA page"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_infeasible_no_indexer():
"""Use the generic layout error when no kpool indexer is present."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBHNC"
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(27):
if i % 4 == 3 or i == 26:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec(
shapes=((512, 1024),), dtypes=(torch.float32,)
)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert kv_cache_spec["layers.0.linear_attn"].page_size_bytes > mla_page
with pytest.raises(NotImplementedError, match="page size"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_prepadded_mamba():
"""Platform-prepadded mamba pages (mamba_page_size_padded hint) must not
disable slot sharing; the layout re-pads them to the MLA page."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
def prepadded_mamba_spec():
return new_mamba_spec(page_size_padded=294_912)
kv_cache_spec, _ = _glm5_like_kv_cache_spec(prepadded_mamba_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
assert prepadded_mamba_spec().page_size_bytes < mla_page
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
attn = [
g
for g in groups
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
and isinstance(g.kv_cache_spec.first_spec, MLAAttentionSpec)
]
assert len(groups) == 5
assert [len(group.layer_names) for group in mamba_groups] == [9, 9, 8, 8]
for group in mamba_groups:
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
assert len(kv_cache_config.kv_cache_tensors) == 56
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_balanced_projection():
"""Round-robin mamba grouping keeps practical PP splits balanced: every
stage's largest projected mamba group slice fits its projected MLA
layers, so per-stage slot tensors all have an MLA owner."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
global_groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
# PP=2 split at transformer layer 22/23.
# Stage 0 of a PP=2 split at transformer layer 22/23: 5 MLA(+indexer)
# layers, projected mamba groups [5, 5, 4, 4].
worker_spec = {n: s for n, s in kv_cache_spec.items() if int(n.split(".")[1]) <= 22}
groups = kv_cache_utils._project_kv_cache_groups_to_worker(
global_groups, worker_spec
)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
rings = [
g
for g in groups
if isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
and isinstance(g.kv_cache_spec.first_spec, CircularBufferSpec)
]
assert [len(group.layer_names) for group in mamba_groups] == [5, 5, 4, 4]
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 5 * mla_page + 5 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
tensors = _tensor_by_layer(kv_cache_config)
# Every projected Mamba slot aliases an MLA owner.
assert tensors["layers.3.attn"].offset == tensors["layers.0.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.1.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.2.linear_attn"].offset
assert tensors["layers.3.attn"].offset == tensors["layers.4.linear_attn"].offset
# The last slot only hosts the two groups with a 5th projected layer.
assert tensors["layers.19.attn"].offset == tensors["layers.21.linear_attn"].offset
assert tensors["layers.19.attn"].offset == tensors["layers.22.linear_attn"].offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_group_count_bump(monkeypatch):
"""PP=4's default partition [11,11,12,11] leaves stage 0 with 2 MLA
layers but a round-robin slice of 3 under the minimum 4 mamba groups;
grouping bumps to 5 groups so every stage's projection keeps sharing
on instead of silently falling back on stage 0."""
from vllm.config import ParallelConfig
from vllm.distributed.utils import get_pp_indices
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=8192),
parallel_config=ParallelConfig(pipeline_parallel_size=4),
)
kv_cache_spec, mamba_layers = _glm5_like_kv_cache_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [g for g in groups if isinstance(g.kv_cache_spec, MambaSpec)]
assert [len(g.layer_names) for g in mamba_groups] == [7, 7, 7, 7, 6]
for k, name in enumerate(mamba_layers):
assert name in mamba_groups[k % 5].layer_names
for rank in range(4):
start, end = get_pp_indices(45, rank, 4)
worker_spec = {
n: s
for n, s in kv_cache_spec.items()
if start <= int(n.split(".")[1]) < end
}
projected = kv_cache_utils._project_kv_cache_groups_to_worker(
groups, worker_spec
mamba = [
g
for g in groups
if isinstance(
getattr(g.kv_cache_spec, "first_spec", g.kv_cache_spec), MambaSpec
)
assert kv_cache_utils._glm5_next_tensor_layout(projected) is not None
def test_get_kv_cache_config_mamba_hybrid_sharing_pp_starved_stage(monkeypatch):
"""Reject PP stages whose Mamba layers have no MLA slot to share."""
from vllm.config import ParallelConfig
monkeypatch.setattr(ModelConfig, "get_total_num_hidden_layers", lambda self: 45)
monkeypatch.setenv("VLLM_PP_LAYER_PARTITION", "3,42")
vllm_config = VllmConfig(
model_config=ModelConfig(max_model_len=8192),
parallel_config=ParallelConfig(pipeline_parallel_size=2),
)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
with pytest.raises(ValueError, match="VLLM_PP_LAYER_PARTITION"):
kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
def test_get_kv_cache_config_mamba_hybrid_sharing_beats_cross_layers_flag():
"""Hybrid slot sharing must take precedence over the experimental
enable_cross_layers_blocks packed layout: generic packing would give the
MLA slots a strided view, breaking the contiguous virtual split."""
model_config = ModelConfig(max_model_len=8192)
kv_transfer_config = KVTransferConfig(
kv_connector="NixlConnector",
kv_role="kv_both",
kv_connector_extra_config={"enable_cross_layers_blocks": "true"},
)
vllm_config = VllmConfig(
model_config=model_config, kv_transfer_config=kv_transfer_config
)
kv_cache_spec, _ = _glm5_like_kv_cache_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
idx_page = kv_cache_spec["layers.3.indexer"].page_size_bytes
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 11 * mla_page + 11 * idx_page
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
assert len(kv_cache_config.kv_cache_tensors) == 56
assert all(
tensor.block_stride in (mla_page, idx_page)
for tensor in kv_cache_config.kv_cache_tensors
)
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_config_mamba_hybrid_sharing_no_indexer():
"""Kimi-Linear-like: MLA without indexer layers, idx_stride == 0."""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
vllm_config.cache_config.kv_cache_layout = "LBNHC"
# 20 mamba + 7 MLA layers -> G = ceil(20 / 7) = 3 groups of [7, 7, 6].
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(27):
if i % 4 == 3 or i == 26:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
mla_page = kv_cache_spec["layers.3.attn"].page_size_bytes
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
mamba_groups = [
group for group in groups if isinstance(group.kv_cache_spec, MambaSpec)
]
assert len(groups) == 4
assert [len(group.layer_names) for group in mamba_groups] == [7, 7, 6]
for group in mamba_groups:
assert group.kv_cache_spec.page_size_bytes == mla_page
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
assert bytes_per_block == 7 * mla_page
assert len(attn) == 1 and len(attn[0].layer_names) == 22
assert len(rings) == 1 and len(rings[0].layer_names) == 11
# 11 * (1.18 MB MLA + 34 KB indexer) per block fits 12 fp32-SSM KDA states.
assert len(mamba) == 3
assert sum(len(g.layer_names) for g in mamba) == 34
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
assert kv_cache_config.num_blocks == 100
# Each group has one tensor descriptor and all groups alias the same slots.
assert len(kv_cache_config.kv_cache_tensors) == 4
tensors = _tensor_by_layer(kv_cache_config)
mla_names = [f"layers.{i}.attn" for i in (*range(3, 27, 4), 26)]
for index, mla_name in enumerate(mla_names):
mla_offset = _layer_offset(tensors[mla_name], mla_name)
for group in mamba_groups:
if index < len(group.layer_names):
mamba_name = group.layer_names[index]
assert _layer_offset(tensors[mamba_name], mamba_name) == mla_offset
assert {t.size for t in kv_cache_config.kv_cache_tensors} == {bytes_per_block * 100}
def test_get_kv_cache_capacity_after_scheduler_unwrap():
"""max_concurrency must survive the scheduler-config unwrap.
Regression for the balanced-mamba hybrid layout:
``generate_scheduler_kv_cache_config`` flattens the MLA
``UniformTypeKVCacheSpecs`` group into a single ``MLAAttentionSpec``, so the
scheduler config holds an MLA spec (page size A) next to ``MambaSpec``
groups (page size B). ``EngineCore._initialize_kv_caches`` calls
``get_kv_cache_capacity`` on that unwrapped config; the uniform-page-size
path used to assert-fail (``assert len(page_sizes) == 1``) on this topology.
"""
model_config = ModelConfig(max_model_len=8192)
vllm_config = VllmConfig(model_config=model_config)
kv_cache_spec: dict[str, KVCacheSpec] = {}
for i in range(45):
if i % 4 == 3:
kv_cache_spec[f"layers.{i}.attn"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=576,
dtype=torch.bfloat16,
)
kv_cache_spec[f"layers.{i}.indexer"] = MLAAttentionSpec(
block_size=1024,
num_kv_heads=1,
head_size=132,
dtype=torch.uint8,
tokens_per_state=16,
)
else:
kv_cache_spec[f"layers.{i}.linear_attn"] = new_mamba_spec()
groups = kv_cache_utils.get_kv_cache_groups(vllm_config, kv_cache_spec)
bytes_per_block = kv_cache_utils._pool_bytes_per_block(groups)
kv_cache_config = kv_cache_utils.get_kv_cache_config_from_groups(
vllm_config, groups, bytes_per_block * 100 + 1
)
scheduler_config = generate_scheduler_kv_cache_config([kv_cache_config])
# Confirm we are exercising the regression path: the MLA group is no longer
# UniformTypeKVCacheSpecs, so sibling specs have differing page sizes.
assert not any(
isinstance(g.kv_cache_spec, UniformTypeKVCacheSpecs)
for g in scheduler_config.kv_cache_groups
)
unwrapped_groups = scheduler_config.kv_cache_groups
expected_max_mem = kv_cache_utils._max_memory_usage_bytes_from_groups(
vllm_config, unwrapped_groups
)
expected_pool = kv_cache_utils._pool_bytes_per_block(unwrapped_groups)
expected_blocks_per_request = (
expected_max_mem + expected_pool - 1
) // expected_pool
_, max_concurrency = get_kv_cache_capacity(vllm_config, scheduler_config)
assert max_concurrency > 0
assert max_concurrency == pytest.approx(
scheduler_config.num_blocks / expected_blocks_per_request
vllm_config, groups, available_memory=2 * 1024**3
)
strides = {t.block_stride for t in kv_cache_config.kv_cache_tensors}
assert len(strides) == 1
(block_stride,) = strides
# Blocks sit a whole number of 32-row MLA pages and 64-state indexer pages
# apart so both caches can be re-paged for their kernels.
assert block_stride % (32 * 576 * 2) == 0
assert block_stride % (64 * 132) == 0
def new_mla_spec(cache_dtype_str=None, block_size: int = 16):
@@ -4483,6 +3993,7 @@ def test_kv_cache_reserves_null_block_for_max_model_len(use_override):
block_size = 16
max_model_len = 512 # needs 512 / 16 = 32 blocks
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
vllm_config.cache_config.kv_cache_layout = "LBNHC"
spec = new_kv_cache_spec(block_size=block_size)
# 32 blocks -> only 31 usable after the null block: one short -> reject.
@@ -4528,6 +4039,7 @@ def test_check_enough_kv_cache_memory_reserves_null_block():
block_size = 16
max_model_len = 512 # needs 512 / 16 = 32 blocks
vllm_config = VllmConfig(model_config=ModelConfig(max_model_len=max_model_len))
vllm_config.cache_config.kv_cache_layout = "LBNHC"
spec = new_kv_cache_spec(block_size=block_size)
# 32 blocks -> only 31 usable after the null block: one short -> reject.
@@ -20,7 +20,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
KpoolTailManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
@@ -29,7 +28,6 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
@@ -406,15 +404,15 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
@pytest.mark.parametrize("record_for_zeroing", [True, False])
def test_external_kpool_tail_zeroing(record_for_zeroing):
"""A transferred ring at index zero skips zeroing even with a local hit."""
spec = KpoolTailSpec(
"""The kpool tail ring is never zeroed, transferred or not: a pool only
reads ring slots its own request wrote."""
spec = CircularBufferSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
sliding_window=4,
)
manager = KpoolTailManager(
manager = CircularBufferManager(
spec,
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
enable_caching=True,
@@ -428,9 +426,7 @@ def test_external_kpool_tail_zeroing(record_for_zeroing):
)
blocks = manager.req_to_blocks["request"]
assert len(blocks) == 1
assert manager.take_new_block_ids() == (
[blocks[0].block_id] if record_for_zeroing else []
)
assert manager.take_new_block_ids() == []
assert manager.allocate_new_blocks("request", 20, 20) == []
assert manager.take_new_block_ids() == []