[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+4 -4
View File
@@ -661,8 +661,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
self.hash_block_size = hash_block_size
self.dcp_world_size = dcp_world_size
# Only groups that participate in prefix caching must satisfy the
# divisibility constraint; groups that opt out (e.g. GLM-5.3-Flash kpool
# tail, block_size=kpool) are scratch buffers and excluded.
# divisibility constraint; groups that opt out (e.g. circular buffers
# such as the GLM-5.3-Flash indexer tail) are scratch and excluded.
cacheable_block_sizes = [
manager.block_size
for manager, group in zip(
@@ -724,8 +724,8 @@ class HybridKVCacheCoordinator(KVCacheCoordinator):
"""
self.attention_groups: list[SpecGroup] = []
for i, g in enumerate(self.kv_cache_config.kv_cache_groups):
# Skip groups that opt out of prefix caching (e.g. GLM-5.3-Flash
# kpool tail): their blocks are per-request scratch, never
# Skip groups that opt out of prefix caching (e.g. circular
# buffers): their blocks are per-request scratch, never
# shareable, so they must not participate in hit lookup (their
# manager-level hooks already no-op). Their slot in the per-group
# hit tuple stays empty.
+41 -313
View File
@@ -10,7 +10,7 @@ from collections import defaultdict
from collections.abc import Callable, Iterable, Iterator, Sequence
from dataclasses import dataclass, field, replace
from functools import partial, reduce
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, cast, overload
from typing import TYPE_CHECKING, Any, NamedTuple, NewType, TypeAlias, overload
from vllm import envs
from vllm.config import VllmConfig
@@ -32,7 +32,6 @@ from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
HiddenStateCacheSpec,
HiSparseHotSpec,
KpoolTailSpec,
KVCacheConfig,
KVCacheGroupSpec,
KVCacheLayout,
@@ -1126,7 +1125,7 @@ def check_enough_kv_cache_memory(
# of the specs since grouping may unify them in-place.
groups = get_kv_cache_groups(vllm_config, dict(kv_cache_spec))
check_memory = (
available_memory - _pool_bytes_per_block(groups)
available_memory - _pool_bytes_per_block(vllm_config, groups)
if groups
else available_memory
)
@@ -1237,14 +1236,17 @@ def may_override_num_blocks(vllm_config: VllmConfig, num_blocks: int) -> int:
return num_blocks
def _pool_bytes_per_block(kv_cache_groups: list[KVCacheGroupSpec]) -> int:
def _pool_bytes_per_block(
vllm_config: VllmConfig, kv_cache_groups: list[KVCacheGroupSpec]
) -> int:
"""Bytes consumed by one block in the worker's shared KV cache pool, mirroring
the divisor used by `get_kv_cache_config_from_groups` to convert
`available_memory` into `num_blocks`. Used to compute the effective KV cache
capacity once `num_gpu_blocks_override` is applied.
"""
return _get_kv_cache_bytes_per_block(
[group for group in kv_cache_groups if not group.host_resident]
[group for group in kv_cache_groups if not group.host_resident],
vllm_config.cache_config.get_resolved_kv_cache_layout(),
)
@@ -1287,205 +1289,6 @@ def _get_kv_cache_groups_uniform_type(
return [KVCacheGroupSpec(list(spec.kv_cache_specs.keys()), spec)]
def _pp_balanced_mamba_group_count(
vllm_config: VllmConfig,
mamba_layer_names: list[str],
mla_layer_names: list[str],
) -> int | None:
"""Return a Mamba group count whose PP projections fit the MLA slots."""
num_groups = cdiv(len(mamba_layer_names), len(mla_layer_names))
pp_size = vllm_config.parallel_config.pipeline_parallel_size
if pp_size == 1:
return num_groups
from vllm.distributed.utils import get_pp_indices
from vllm.model_executor.models.utils import extract_layer_index
total_layers = vllm_config.model_config.get_total_num_hidden_layers()
mamba_indices = [extract_layer_index(name) for name in mamba_layer_names]
mla_indices = [extract_layer_index(name) for name in mla_layer_names]
for rank in range(pp_size):
start, end = get_pp_indices(total_layers, rank, pp_size)
num_mamba = sum(start <= index < end for index in mamba_indices)
num_mla = sum(start <= index < end for index in mla_indices)
if not num_mamba:
continue
if not num_mla:
return None
num_groups = max(num_groups, cdiv(num_mamba, num_mla))
return num_groups
def _get_kv_cache_groups_glm5_next(
vllm_config: VllmConfig,
kv_cache_spec: dict[str, KVCacheSpec],
) -> list[KVCacheGroupSpec] | None:
"""Build GLM-5.3-Flash groups with Mamba/MLA and tail/indexer aliasing."""
mamba_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, MambaSpec)
}
tail_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if isinstance(spec, KpoolTailSpec)
}
attn_specs = {
name: spec
for name, spec in kv_cache_spec.items()
if not isinstance(spec, (MambaSpec, KpoolTailSpec))
}
if not mamba_specs or not all(
type(spec) is MLAAttentionSpec for spec in attn_specs.values()
):
return None
mla_specs = cast(dict[str, MLAAttentionSpec], attn_specs)
idx_pages = {
spec.page_size_bytes for spec in mla_specs.values() if spec.tokens_per_state > 1
}
if not idx_pages:
return None
assert all(spec.page_size_padded is None for spec in mla_specs.values())
assert len(idx_pages) == 1
mla_names = [name for name, spec in mla_specs.items() if spec.tokens_per_state == 1]
mla_pages = {mla_specs[name].page_size_bytes for name in mla_names}
assert len(mla_pages) == 1
mla_page = mla_pages.pop()
uniform_spec = UniformTypeKVCacheSpecs.from_specs(attn_specs)
assert uniform_spec is not None
tail_group: KVCacheGroupSpec | None = None
if tail_specs:
idx_page = next(iter(idx_pages))
padded_tail_specs: dict[str, KVCacheSpec] = {
name: replace(spec, page_size_padded=idx_page)
for name, spec in tail_specs.items()
}
tail_uniform = UniformTypeKVCacheSpecs.from_specs(padded_tail_specs)
assert tail_uniform is not None
tail_group = KVCacheGroupSpec(list(padded_tail_specs), tail_uniform)
any_mamba = next(iter(mamba_specs.values()))
assert all(spec == any_mamba for spec in mamba_specs.values())
if any_mamba.real_page_size_bytes > mla_page:
raise ValueError(
f"the mamba state page ({any_mamba.real_page_size_bytes} bytes) "
f"does not fit the MLA page ({mla_page} bytes); increase tensor "
"parallelism or use a wider KV cache dtype"
)
padded_specs: dict[str, KVCacheSpec] = {
name: replace(any_mamba, page_size_padded=mla_page) for name in mamba_specs
}
num_groups = _pp_balanced_mamba_group_count(
vllm_config, list(mamba_specs), mla_names
)
if num_groups is None:
raise ValueError(
"a pipeline stage has mamba layers but no MLA layer to share "
"slots with; realign the stage boundaries (VLLM_PP_LAYER_PARTITION)"
)
mamba_grouped_names: list[list[str]] = [[] for _ in range(num_groups)]
for index, name in enumerate(mamba_specs):
mamba_grouped_names[index % num_groups].append(name)
return (
[KVCacheGroupSpec(list(attn_specs), uniform_spec)]
+ ([tail_group] if tail_group is not None else [])
+ create_kv_cache_group_specs(padded_specs, mamba_grouped_names)
)
def _glm5_next_tensor_layout(
kv_cache_groups: list[KVCacheGroupSpec],
) -> (
tuple[
KVCacheGroupSpec,
list[KVCacheGroupSpec],
list[str],
list[str],
int,
int,
list[str],
int,
]
| None
):
"""Recognize the GLM-5.3-Flash grouping after optional PP projection."""
uniform_groups = [
group
for group in kv_cache_groups
if isinstance(group.kv_cache_spec, UniformTypeKVCacheSpecs)
]
mamba_groups = [
group for group in kv_cache_groups if isinstance(group.kv_cache_spec, MambaSpec)
]
attn_group: KVCacheGroupSpec | None = None
tail_group: KVCacheGroupSpec | None = None
for group in uniform_groups:
inner = cast(UniformTypeKVCacheSpecs, group.kv_cache_spec).kv_cache_specs
if all(type(spec) is MLAAttentionSpec for spec in inner.values()):
attn_group = group
elif all(isinstance(spec, KpoolTailSpec) for spec in inner.values()):
tail_group = group
if attn_group is None or not mamba_groups:
return None
if len(uniform_groups) + len(mamba_groups) != len(kv_cache_groups):
return None
attn_uniform = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
mla_inner = cast(dict[str, MLAAttentionSpec], attn_uniform.kv_cache_specs)
if not all(
type(spec) is MLAAttentionSpec and spec.page_size_padded is None
for spec in mla_inner.values()
):
return None
mla_names = [
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state == 1
]
idx_names = [
name for name in attn_group.layer_names if mla_inner[name].tokens_per_state > 1
]
mla_pages = {mla_inner[name].page_size_bytes for name in mla_names}
idx_pages = {mla_inner[name].page_size_bytes for name in idx_names}
if len(mla_pages) != 1 or len(idx_pages) != 1:
return None
mla_page = mla_pages.pop()
idx_page = idx_pages.pop()
if any(group.kv_cache_spec.page_size_bytes != mla_page for group in mamba_groups):
return None
tail_names: list[str] = []
tail_page = 0
if tail_group is not None:
tail_names = list(tail_group.layer_names)
tail_inner = cast(
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
).kv_cache_specs
tail_pages = {
cast(KpoolTailSpec, spec).unpadded_page_size_bytes
for spec in tail_inner.values()
}
if len(tail_pages) != 1 or len(tail_names) != len(idx_names):
return None
tail_page = tail_pages.pop()
if tail_page > idx_page:
return None
return (
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
tail_page,
)
def unify_kv_cache_spec_page_size(
kv_cache_spec: dict[str, KVCacheSpec],
) -> dict[str, KVCacheSpec]:
@@ -1709,14 +1512,24 @@ def _get_per_layer_spec(
return spec
def _get_layer_block_stride(spec: KVCacheSpec) -> int:
"""A layer-outer block stride: the page padded to its alignment."""
return round_up(spec.page_size_bytes, spec.get_block_stride_alignment())
def _get_kv_cache_bytes_per_block(
kv_cache_groups: list[KVCacheGroupSpec],
kv_cache_groups: list[KVCacheGroupSpec], layout: KVCacheLayout
) -> int:
"""Return the largest cache group's bytes per block."""
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
_, _, mla_names, idx_names, mla_page, idx_page, _, _ = glm5_layout
return len(mla_names) * mla_page + len(idx_names) * idx_page
if not layout.is_block_outermost:
# Each layer's block stride is its own padded page.
return max(
sum(
_get_layer_block_stride(_get_per_layer_spec(group, layer_name))
for layer_name in group.layer_names
)
for group in kv_cache_groups
)
bytes_per_block = max(
sum(
_get_per_layer_spec(group, layer_name).page_size_bytes
@@ -1731,7 +1544,7 @@ def _get_kv_cache_bytes_per_block(
if isinstance(group.kv_cache_spec, HiSparseHotSpec)
]
alignments.extend(
_get_per_layer_spec(group, layer_name).block_stride_alignment or 1
_get_per_layer_spec(group, layer_name).get_block_stride_alignment()
for group in kv_cache_groups
for layer_name in group.layer_names
)
@@ -1806,72 +1619,9 @@ def get_kv_cache_config_from_groups(
vllm_config, kv_cache_groups, available_memory, host_budget
)
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
_,
) = glm5_layout
bytes_per_block = len(mla_names) * mla_page + len(idx_names) * idx_page
num_blocks = may_override_num_blocks(
vllm_config, available_memory // bytes_per_block
)
size = bytes_per_block * num_blocks
attn_specs = cast(
UniformTypeKVCacheSpecs, attn_group.kv_cache_spec
).kv_cache_specs
kv_cache_tensors: list[KVCacheTensor] = []
def add_tensor(layer_name: str, spec: KVCacheSpec, offset: int) -> None:
kv_cache_tensors.append(
KVCacheTensor(
size=size,
layers=[layer_name],
layer_stride=spec.page_size_bytes * num_blocks,
block_stride=spec.page_size_bytes,
offset=offset,
)
)
for index, mla_name in enumerate(mla_names):
offset = index * mla_page * num_blocks
add_tensor(mla_name, attn_specs[mla_name], offset)
for group in mamba_groups:
if index < len(group.layer_names):
add_tensor(group.layer_names[index], group.kv_cache_spec, offset)
idx_base = len(mla_names) * mla_page * num_blocks
for index, idx_name in enumerate(idx_names):
offset = idx_base + index * idx_page * num_blocks
add_tensor(idx_name, attn_specs[idx_name], offset)
if tail_names:
tail_name = tail_names[index]
tail_group = next(
group for group in kv_cache_groups if tail_name in group.layer_names
)
tail_specs = cast(
UniformTypeKVCacheSpecs, tail_group.kv_cache_spec
).kv_cache_specs
add_tensor(tail_name, tail_specs[tail_name], offset)
return KVCacheConfig(
num_blocks=num_blocks,
kv_cache_tensors=kv_cache_tensors,
kv_cache_groups=kv_cache_groups,
prefix_cache_retention_interval=(
vllm_config.cache_config.prefix_cache_retention_interval
),
)
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
validate_kv_cache_layout(layout, kv_cache_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups)
bytes_per_block = _get_kv_cache_bytes_per_block(kv_cache_groups, layout)
interleaved_block_stride = bytes_per_block if layout.is_block_outermost else None
num_blocks = available_memory // bytes_per_block
@@ -1901,17 +1651,17 @@ def get_kv_cache_config_from_groups(
byte_offset = 0
for spec, layer_names in layers_by_spec.items():
block_stride = interleaved_block_stride
padded_page = _get_layer_block_stride(spec)
if block_stride is None and padded_page != spec.page_size_bytes:
assert layout.is_block_compact, f"{layout.name} cannot pad pages"
block_stride = padded_page
layer_stride, block_stride, _, _, _ = compute_layout_strides(
spec,
num_blocks,
len(layer_names),
layout,
fixed_strides=(None, interleaved_block_stride, None, None, None),
)
offset = (
byte_offset
* max(layer_stride, spec.page_size_bytes)
// spec.page_size_bytes
fixed_strides=(None, block_stride, None, None, None),
)
kv_cache_tensors.append(
KVCacheTensor(
@@ -1919,10 +1669,13 @@ def get_kv_cache_config_from_groups(
layers=layer_names,
layer_stride=layer_stride,
block_stride=block_stride,
offset=offset,
offset=byte_offset,
)
)
byte_offset += len(layer_names) * spec.page_size_bytes
# Block-outer layers share each block; layer-outer ones own regions.
byte_offset += len(layer_names) * (
spec.page_size_bytes if layout.is_block_outermost else layer_stride
)
return KVCacheConfig(
num_blocks=num_blocks,
@@ -2438,9 +2191,6 @@ def get_kv_cache_groups(
# full attention, or all layers are sliding window attention with the
# same window size). Put all layers into one group.
return _get_kv_cache_groups_uniform_type(uniform_spec)
elif glm5_groups := _get_kv_cache_groups_glm5_next(vllm_config, kv_cache_spec):
return glm5_groups
# Hidden-state layers use their own block table and must not be absorbed
# into a compatible attention bucket.
hidden_specs = {
@@ -2594,31 +2344,7 @@ def _max_memory_usage_bytes_from_groups(
if not kv_cache_groups:
return 0
if (glm5_layout := _glm5_next_tensor_layout(kv_cache_groups)) is not None:
(
attn_group,
mamba_groups,
mla_names,
idx_names,
mla_page,
idx_page,
tail_names,
_,
) = glm5_layout
uniform_spec = cast(UniformTypeKVCacheSpecs, attn_group.kv_cache_spec)
total_blocks = uniform_spec.max_memory_usage_pages(vllm_config)
total_blocks += sum(
cdiv(
group.kv_cache_spec.max_memory_usage_bytes(vllm_config),
group.kv_cache_spec.page_size_bytes,
)
for group in mamba_groups
)
if tail_names:
total_blocks += 1
return total_blocks * (len(mla_names) * mla_page + len(idx_names) * idx_page)
bytes_per_block = _pool_bytes_per_block(kv_cache_groups)
bytes_per_block = _pool_bytes_per_block(vllm_config, kv_cache_groups)
total_blocks = 0
for group in kv_cache_groups:
spec = group.kv_cache_spec
@@ -2874,7 +2600,7 @@ def get_kv_cache_configs(
if not groups:
adjusted_memory.append(avail_mem)
continue
bytes_per_block = _pool_bytes_per_block(groups)
bytes_per_block = _pool_bytes_per_block(vllm_config, groups)
logger.info(
"Overriding num_gpu_blocks=%d with num_gpu_blocks_override=%d",
avail_mem // bytes_per_block,
@@ -2887,7 +2613,7 @@ def get_kv_cache_configs(
# the capacity check both plan against usable blocks. Allocation below
# still uses the full memory.
check_memory = [
avail_mem - _pool_bytes_per_block(groups) if groups else avail_mem
avail_mem - _pool_bytes_per_block(vllm_config, groups) if groups else avail_mem
for groups, avail_mem in zip(projected_groups_per_worker, available_memory)
]
@@ -2931,7 +2657,9 @@ def get_kv_cache_configs(
# strides and offsets stay consistent with the shrunken allocation.
groups = kv_cache_config.kv_cache_groups
kv_cache_configs[i] = get_kv_cache_config_from_groups(
vllm_config, groups, min_num_blocks * _pool_bytes_per_block(groups)
vllm_config,
groups,
min_num_blocks * _pool_bytes_per_block(vllm_config, groups),
)
for kv_cache_config in kv_cache_configs:
@@ -7,7 +7,6 @@ from collections.abc import Sequence
from typing import TYPE_CHECKING, ClassVar
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_utils import (
@@ -27,7 +26,6 @@ from vllm.v1.kv_cache_interface import (
HiddenStateCacheSpec,
HiSparseHotSpec,
HiSparseResidentSpec,
KpoolTailSpec,
KVCacheGroupRole,
KVCacheSpec,
MambaSpec,
@@ -45,8 +43,6 @@ from vllm.v1.request import Request
if TYPE_CHECKING:
from vllm.v1.hisparse.coordinator import HiSparseCoordinator
logger = init_logger(__name__)
class SingleTypeKVCacheManager(ABC):
"""An abstract base class for a manager that handle the kv cache management
@@ -1299,10 +1295,6 @@ class CircularBufferManager(FullAttentionManager):
return 0
class KpoolTailManager(CircularBufferManager):
"""One-block circular scratch manager for ``KpoolTailSpec``."""
class ChunkedLocalAttentionManager(SingleTypeKVCacheManager):
def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None:
super().__init__(kv_cache_spec, **kwargs)
@@ -2687,11 +2679,6 @@ def register_all_kvcache_specs(vllm_config):
SlidingWindowManager,
uniform_type_base_spec=SlidingWindowMLASpec,
)
KVCacheSpecRegistry.register(
KpoolTailSpec,
KpoolTailManager,
uniform_type_base_spec=KpoolTailSpec,
)
KVCacheSpecRegistry.register(
MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec