[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+207 -45
View File
@@ -3,7 +3,7 @@
import math
from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field
from dataclasses import dataclass, field, replace
from itertools import product as iprod
from typing import Any
@@ -24,6 +24,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.attention.backend import (
AttentionBackend,
AttentionMetadataBuilder,
CommonAttentionMetadata,
MultipleOf,
)
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
@@ -35,7 +36,6 @@ from vllm.v1.kv_cache_interface import (
KVCacheLayout,
KVCacheSpec,
MambaSpec,
MLAAttentionSpec,
UniformTypeKVCacheSpecs,
create_kv_cache_views,
)
@@ -166,6 +166,9 @@ class KVBlockZeroer:
kv = static_forward_context[layer_name].kv_cache
if not isinstance(kv, torch.Tensor):
continue
if group.kernel_block_stride:
# Bound views address kernel blocks; zero by manager block.
kv = group.map_kv_cache_to_manager_blocks(kv, num_blocks)
if kv.device.type != self.device.type:
continue
dp = kv.data_ptr()
@@ -255,12 +258,22 @@ class KVBlockZeroer:
self.zero_block_ids([0])
@dataclass
@dataclass(eq=False)
class AttentionGroup:
backend: type[AttentionBackend]
layer_names: list[str]
kv_cache_spec: KVCacheSpec
kv_cache_group_id: int
# Set when the backend needs blocks smaller than the KV cache manager's: its
# block tables and KV cache views then address kernel blocks, kernel block j
# of block b being b * kernel_block_stride + j. The stride is set when a
# packed KV cache is bound (other layers' pages sit between its blocks);
# otherwise it stays 0 and the kernel blocks per block are used.
kernel_block_size: int | None = None
kernel_block_stride: int = 0
# Persistent per ubatch: CUDA graphs capture the mapped block table.
kernel_block_table: torch.Tensor | None = None
kernel_block_offsets: torch.Tensor | None = None
# When ubatching is enabled we will have a metadata builder for each ubatch
# so that if they use internal persistent buffers for cudagraphs, and they
# won't have to worry about conflicting with the other ubatches.
@@ -275,27 +288,44 @@ class AttentionGroup:
kernel_block_size: int | None = None,
num_metadata_builders: int = 1,
):
spec = self.kv_cache_spec
if kernel_block_size is not None and isinstance(spec, AttentionSpec):
# A backend in a packed KV cache group may need smaller blocks than
# the group is split into.
kernel_block_size = select_common_block_size(
kernel_block_size, [self.backend], [spec]
)
if kernel_block_size != spec.block_size:
self.kernel_block_size = kernel_block_size
if kernel_block_size is None:
kv_cache_spec_builder = self.kv_cache_spec
elif (
isinstance(self.kv_cache_spec, MLAAttentionSpec)
and self.kv_cache_spec.storage_block_size is not None
):
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
self.kv_cache_spec.storage_block_size
)
else:
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
kernel_block_size
)
builder_cls = self.backend.get_builder_cls()
builder_kwargs = {}
if builder_cls.requires_block_table_width:
if builder_cls.requires_block_table_width or self.kernel_block_size is not None:
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
vllm_config, vllm_config.model_config.max_model_len
)
builder_kwargs["block_table_width"] = get_block_table_width(
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
width = get_block_table_width(max_num_blocks, self.kv_cache_spec.block_size)
if kernel_block_size is not None:
width *= self.kv_cache_spec.block_size // kernel_block_size
if builder_cls.requires_block_table_width:
builder_kwargs["block_table_width"] = width
if self.kernel_block_size is not None:
self.kernel_block_table = torch.zeros(
num_metadata_builders,
vllm_config.scheduler_config.max_num_seqs,
width,
dtype=torch.int32,
device=device,
)
self.kernel_block_offsets = torch.arange(
spec.block_size // self.kernel_block_size,
dtype=torch.int32,
device=device,
)
self.metadata_builders = [
builder_cls(
@@ -307,9 +337,109 @@ class AttentionGroup:
)
for _ in range(num_metadata_builders)
]
if kernel_block_size is not None:
for builder in self.metadata_builders:
builder.set_kernel_block_size(kernel_block_size)
def build_metadata(
self,
common_attn_metadata: CommonAttentionMetadata,
ubatch_idx: int = 0,
common_prefix_len: int = 0,
**kwargs,
) -> Any:
return self.get_metadata_builder(ubatch_idx).build(
common_prefix_len,
self._with_kernel_block_table(common_attn_metadata, ubatch_idx),
**kwargs,
)
def build_metadata_for_cudagraph_capture(
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
) -> Any:
return self.get_metadata_builder(ubatch_idx).build_for_cudagraph_capture(
self._with_kernel_block_table(common_attn_metadata, ubatch_idx)
)
def build_metadata_for_drafting(
self, common_attn_metadata: CommonAttentionMetadata, draft_index: int
) -> Any:
return self.get_metadata_builder().build_for_drafting(
self._with_kernel_block_table(common_attn_metadata), draft_index
)
def _with_kernel_block_table(
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
) -> CommonAttentionMetadata:
if self.kernel_block_size is None:
return common_attn_metadata
return replace(
common_attn_metadata,
block_table_tensor=self.map_to_kernel_block_table(
common_attn_metadata.block_table_tensor, ubatch_idx
),
)
def map_to_kernel_block_table(
self, block_table: torch.Tensor, ubatch_idx: int = 0
) -> torch.Tensor:
if self.kernel_block_size is None:
return block_table
assert self.kernel_block_table is not None
blocks_per_kv_block = self.kv_cache_spec.block_size // self.kernel_block_size
rows, cols = block_table.shape
out = self.kernel_block_table[ubatch_idx, :rows, : cols * blocks_per_kv_block]
torch.add(
self.kernel_block_offsets,
block_table.unsqueeze(-1),
alpha=self.kernel_block_stride or blocks_per_kv_block,
out=out.unflatten(1, (cols, -1)),
)
return out
def map_kv_cache_to_kernel_blocks(self, kv_cache: torch.Tensor) -> torch.Tensor:
"""Views a packed cache in manager blocks as kernel blocks."""
assert self.kernel_block_size is not None
spec = self.kv_cache_spec
kernel_rows = spec.get_num_kernel_states(self.kernel_block_size)
if kv_cache.shape[-2] == kernel_rows:
return kv_cache
# Token and pooled caches are written through manager-block slot
# mappings, so only compressed caches (written from their block table)
# can be mapped.
if (
spec.tokens_per_state <= 1
or kv_cache.shape[1] != 1
or kv_cache.stride(0) % (kernel_rows * kv_cache.stride(-2))
):
raise ValueError(
f"{self.backend.get_name()} needs {self.kernel_block_size}-token "
f"kernel blocks, which packed {spec.block_size}-token KV cache "
f"blocks cannot be split into. Reduce --block-size to "
f"{self.kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
"layer-compact layout (e.g. LBNHC)."
)
# Kernel block j of block b is b * kernel_block_stride + j. The view
# spans the other layers' pages between blocks (never addressing them),
# which ``view`` cannot express.
page_stride = kernel_rows * kv_cache.stride(-2)
self.kernel_block_stride = kv_cache.stride(0) // page_stride
num_pages = (kv_cache.shape[0] - 1) * self.kernel_block_stride + (
kv_cache.shape[-2] // kernel_rows
)
return kv_cache.as_strided(
(num_pages, *kv_cache.shape[1:-2], kernel_rows, kv_cache.shape[-1]),
(page_stride, *kv_cache.stride()[1:]),
)
def map_kv_cache_to_manager_blocks(
self, kv_cache: torch.Tensor, num_blocks: int
) -> torch.Tensor:
"""Views a packed cache in kernel blocks as manager blocks."""
assert self.kernel_block_stride
spec = self.kv_cache_spec
rows = spec.get_num_kernel_states(spec.block_size)
return kv_cache.as_strided(
(num_blocks, *kv_cache.shape[1:-2], rows, kv_cache.shape[-1]),
(kv_cache.stride(0) * self.kernel_block_stride, *kv_cache.stride()[1:]),
)
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
assert len(self.metadata_builders) > ubatch_id
@@ -327,17 +457,48 @@ class AttentionGroup:
self.get_metadata_builder().update_draft_decode_metadata(metadata)
def map_kv_caches_to_kernel_blocks(
kv_caches: dict[str, torch.Tensor], attn_groups: Iterable[AttentionGroup]
) -> dict[str, torch.Tensor]:
"""The views to bind to layers. The manager views in ``kv_caches`` stay for
KV connectors."""
mapped = dict(kv_caches)
for group in attn_groups:
if group.kernel_block_size is not None:
mapped.update(
(name, group.map_kv_cache_to_kernel_blocks(kv_caches[name]))
for name in group.layer_names
if name in kv_caches
)
return mapped
def customize_attention_spec(
backend: type[AttentionBackend], spec: AttentionSpec
) -> AttentionSpec:
"""The backend's spec. Blocks larger than every kernel block size a backend
takes are split into kernel blocks, which packed blocks must keep whole."""
spec = backend.customize_spec(spec)
sizes = backend.get_supported_kernel_block_sizes(spec)
fixed_sizes = [s for s in sizes if isinstance(s, int)]
if spec.block_stride_alignment is None and sizes and fixed_sizes == sizes:
spec = replace(spec, block_stride_alignment=MultipleOf(max(fixed_sizes)))
return spec
def _block_size_is_supported(
backends: list[type[AttentionBackend]], block_size: int
backends: list[type[AttentionBackend]],
block_size: int,
specs: Sequence[KVCacheSpec | None],
) -> bool:
"""Check if the block size is supported by all backends.
An exact ``int`` declaration must match exactly; a ``MultipleOf``
declaration accepts any multiple of its base.
"""
for backend in backends:
for backend, spec in zip(backends, specs, strict=True):
is_supported = False
for supported_size in backend.get_supported_kernel_block_sizes():
for supported_size in backend.get_supported_kernel_block_sizes(spec):
if isinstance(supported_size, int):
if block_size == supported_size:
is_supported = True
@@ -354,6 +515,7 @@ def _block_size_is_supported(
def select_common_block_size(
kv_manager_block_size: int,
backends: list[type[AttentionBackend]],
kv_cache_specs: Sequence[KVCacheSpec | None] | None = None,
) -> int:
"""Select a block size that is supported by all backends and is a factor of
kv_manager_block_size.
@@ -364,6 +526,7 @@ def select_common_block_size(
Args:
kv_manager_block_size: Block size of KV cache.
backends: List of attention backend classes.
kv_cache_specs: Per-backend cache specs for spec-specific constraints.
Returns:
The selected block size.
@@ -372,25 +535,27 @@ def select_common_block_size(
ValueError: If no valid block size found.
"""
if _block_size_is_supported(backends, kv_manager_block_size):
specs = kv_cache_specs or [None] * len(backends)
if _block_size_is_supported(backends, kv_manager_block_size, specs):
return kv_manager_block_size
# MultipleOf constraints also accept the manager size if they accept a divisor.
# Any remaining candidate must therefore be an explicit size from a backend.
candidates = {
size
for backend in backends
for size in backend.get_supported_kernel_block_sizes()
for backend, spec in zip(backends, specs, strict=True)
for size in backend.get_supported_kernel_block_sizes(spec)
if isinstance(size, int) and kv_manager_block_size % size == 0
}
for size in sorted(candidates, reverse=True):
if _block_size_is_supported(backends, size):
if _block_size_is_supported(backends, size, specs):
return size
raise ValueError(
f"No common block size for {kv_manager_block_size} ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends
f"{b.get_name()}: {b.get_supported_kernel_block_sizes(s)}"
for b, s in zip(backends, specs, strict=True)
)
+ ")."
)
@@ -450,9 +615,6 @@ def allocate_kv_cache(
kernel_block_size = None
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
kernel_block_size = kernel_block_sizes[group_id]
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
kernel_block_size = spec.storage_block_size
views = create_kv_cache_views(
buf,
spec,
@@ -497,24 +659,20 @@ def prepare_kernel_block_sizes(
# This is an attention backend that supports virtual block splitting.
kv_manager_block_size = kv_cache_group.kv_cache_spec.block_size
group_backends = [g.backend for g in attn_groups[kv_cache_gid]]
storage_block_size = (
kv_cache_spec.storage_block_size
if isinstance(kv_cache_spec, MLAAttentionSpec)
else None
)
if storage_block_size is not None and _block_size_is_supported(
group_backends, storage_block_size
):
# Storage-block specs (e.g. the GLM-5.3-Flash kpool indexer
# cache) address the cache in pool pages, and every other
# consumer (cache views, metadata builders, hisparse) already
# uses storage_block_size as the kernel block. Fall back to the
# backend vote when the group's backends do not accept it.
selected_kernel_size = storage_block_size
else:
selected_kernel_size = select_common_block_size(
kv_manager_block_size, group_backends
group_specs = [g.kv_cache_spec for g in attn_groups[kv_cache_gid]]
# Block-outermost packing puts other layers' pages between the
# blocks, so they are never split group-wide.
selected_kernel_size = (
kv_manager_block_size
if any(
t.block_stride > t.layer_stride
for t in kv_cache_config.kv_cache_tensors
if not set(t.layers).isdisjoint(kv_cache_group.layer_names)
)
else select_common_block_size(
kv_manager_block_size, group_backends, group_specs
)
)
kernel_block_sizes.append(selected_kernel_size)
elif isinstance(kv_cache_spec, MambaSpec):
# This is likely Mamba or other non-attention cache, no splitting.
@@ -656,6 +814,7 @@ def bind_kv_cache(
runner_kv_caches: list[torch.Tensor],
num_attn_module: int = 1,
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
layer_kv_caches: dict[str, torch.Tensor] | None = None,
) -> None:
"""Bind the allocated KV cache to both ModelRunner and forward context so
that the KV cache can be used in the forward pass.
@@ -674,6 +833,9 @@ def bind_kv_cache(
runner_kv_caches: The kv_cache declared by ModelRunner.
kv_cache_groups: The KV cache groups of the model, used to resolve
layers that share a KV cache.
layer_kv_caches: The views to bind to layers when they differ from
kv_caches (e.g. kernel-block views). The runner keeps kv_caches,
which are addressed by scheduler block IDs.
"""
# Bind kv_caches to ModelRunner
@@ -699,7 +861,7 @@ def bind_kv_cache(
runner_kv_caches.append(kv_caches[layer_name])
bind_kv_cache_to_layers(
kv_caches, forward_context, num_attn_module, kv_cache_groups
layer_kv_caches or kv_caches, forward_context, num_attn_module, kv_cache_groups
)