[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
+207
-45
@@ -3,7 +3,7 @@
|
||||
import math
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import product as iprod
|
||||
from typing import Any
|
||||
|
||||
@@ -24,6 +24,7 @@ from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import (
|
||||
AttentionBackend,
|
||||
AttentionMetadataBuilder,
|
||||
CommonAttentionMetadata,
|
||||
MultipleOf,
|
||||
)
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
@@ -35,7 +36,6 @@ from vllm.v1.kv_cache_interface import (
|
||||
KVCacheLayout,
|
||||
KVCacheSpec,
|
||||
MambaSpec,
|
||||
MLAAttentionSpec,
|
||||
UniformTypeKVCacheSpecs,
|
||||
create_kv_cache_views,
|
||||
)
|
||||
@@ -166,6 +166,9 @@ class KVBlockZeroer:
|
||||
kv = static_forward_context[layer_name].kv_cache
|
||||
if not isinstance(kv, torch.Tensor):
|
||||
continue
|
||||
if group.kernel_block_stride:
|
||||
# Bound views address kernel blocks; zero by manager block.
|
||||
kv = group.map_kv_cache_to_manager_blocks(kv, num_blocks)
|
||||
if kv.device.type != self.device.type:
|
||||
continue
|
||||
dp = kv.data_ptr()
|
||||
@@ -255,12 +258,22 @@ class KVBlockZeroer:
|
||||
self.zero_block_ids([0])
|
||||
|
||||
|
||||
@dataclass
|
||||
@dataclass(eq=False)
|
||||
class AttentionGroup:
|
||||
backend: type[AttentionBackend]
|
||||
layer_names: list[str]
|
||||
kv_cache_spec: KVCacheSpec
|
||||
kv_cache_group_id: int
|
||||
# Set when the backend needs blocks smaller than the KV cache manager's: its
|
||||
# block tables and KV cache views then address kernel blocks, kernel block j
|
||||
# of block b being b * kernel_block_stride + j. The stride is set when a
|
||||
# packed KV cache is bound (other layers' pages sit between its blocks);
|
||||
# otherwise it stays 0 and the kernel blocks per block are used.
|
||||
kernel_block_size: int | None = None
|
||||
kernel_block_stride: int = 0
|
||||
# Persistent per ubatch: CUDA graphs capture the mapped block table.
|
||||
kernel_block_table: torch.Tensor | None = None
|
||||
kernel_block_offsets: torch.Tensor | None = None
|
||||
# When ubatching is enabled we will have a metadata builder for each ubatch
|
||||
# so that if they use internal persistent buffers for cudagraphs, and they
|
||||
# won't have to worry about conflicting with the other ubatches.
|
||||
@@ -275,27 +288,44 @@ class AttentionGroup:
|
||||
kernel_block_size: int | None = None,
|
||||
num_metadata_builders: int = 1,
|
||||
):
|
||||
spec = self.kv_cache_spec
|
||||
if kernel_block_size is not None and isinstance(spec, AttentionSpec):
|
||||
# A backend in a packed KV cache group may need smaller blocks than
|
||||
# the group is split into.
|
||||
kernel_block_size = select_common_block_size(
|
||||
kernel_block_size, [self.backend], [spec]
|
||||
)
|
||||
if kernel_block_size != spec.block_size:
|
||||
self.kernel_block_size = kernel_block_size
|
||||
if kernel_block_size is None:
|
||||
kv_cache_spec_builder = self.kv_cache_spec
|
||||
elif (
|
||||
isinstance(self.kv_cache_spec, MLAAttentionSpec)
|
||||
and self.kv_cache_spec.storage_block_size is not None
|
||||
):
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
self.kv_cache_spec.storage_block_size
|
||||
)
|
||||
else:
|
||||
kv_cache_spec_builder = self.kv_cache_spec.copy_with_new_block_size(
|
||||
kernel_block_size
|
||||
)
|
||||
builder_cls = self.backend.get_builder_cls()
|
||||
builder_kwargs = {}
|
||||
if builder_cls.requires_block_table_width:
|
||||
if builder_cls.requires_block_table_width or self.kernel_block_size is not None:
|
||||
max_num_blocks = self.kv_cache_spec.max_num_blocks_per_req(
|
||||
vllm_config, vllm_config.model_config.max_model_len
|
||||
)
|
||||
builder_kwargs["block_table_width"] = get_block_table_width(
|
||||
max_num_blocks, self.kv_cache_spec.block_size, kernel_block_size
|
||||
width = get_block_table_width(max_num_blocks, self.kv_cache_spec.block_size)
|
||||
if kernel_block_size is not None:
|
||||
width *= self.kv_cache_spec.block_size // kernel_block_size
|
||||
if builder_cls.requires_block_table_width:
|
||||
builder_kwargs["block_table_width"] = width
|
||||
if self.kernel_block_size is not None:
|
||||
self.kernel_block_table = torch.zeros(
|
||||
num_metadata_builders,
|
||||
vllm_config.scheduler_config.max_num_seqs,
|
||||
width,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.kernel_block_offsets = torch.arange(
|
||||
spec.block_size // self.kernel_block_size,
|
||||
dtype=torch.int32,
|
||||
device=device,
|
||||
)
|
||||
self.metadata_builders = [
|
||||
builder_cls(
|
||||
@@ -307,9 +337,109 @@ class AttentionGroup:
|
||||
)
|
||||
for _ in range(num_metadata_builders)
|
||||
]
|
||||
if kernel_block_size is not None:
|
||||
for builder in self.metadata_builders:
|
||||
builder.set_kernel_block_size(kernel_block_size)
|
||||
|
||||
def build_metadata(
|
||||
self,
|
||||
common_attn_metadata: CommonAttentionMetadata,
|
||||
ubatch_idx: int = 0,
|
||||
common_prefix_len: int = 0,
|
||||
**kwargs,
|
||||
) -> Any:
|
||||
return self.get_metadata_builder(ubatch_idx).build(
|
||||
common_prefix_len,
|
||||
self._with_kernel_block_table(common_attn_metadata, ubatch_idx),
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
def build_metadata_for_cudagraph_capture(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
|
||||
) -> Any:
|
||||
return self.get_metadata_builder(ubatch_idx).build_for_cudagraph_capture(
|
||||
self._with_kernel_block_table(common_attn_metadata, ubatch_idx)
|
||||
)
|
||||
|
||||
def build_metadata_for_drafting(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, draft_index: int
|
||||
) -> Any:
|
||||
return self.get_metadata_builder().build_for_drafting(
|
||||
self._with_kernel_block_table(common_attn_metadata), draft_index
|
||||
)
|
||||
|
||||
def _with_kernel_block_table(
|
||||
self, common_attn_metadata: CommonAttentionMetadata, ubatch_idx: int = 0
|
||||
) -> CommonAttentionMetadata:
|
||||
if self.kernel_block_size is None:
|
||||
return common_attn_metadata
|
||||
return replace(
|
||||
common_attn_metadata,
|
||||
block_table_tensor=self.map_to_kernel_block_table(
|
||||
common_attn_metadata.block_table_tensor, ubatch_idx
|
||||
),
|
||||
)
|
||||
|
||||
def map_to_kernel_block_table(
|
||||
self, block_table: torch.Tensor, ubatch_idx: int = 0
|
||||
) -> torch.Tensor:
|
||||
if self.kernel_block_size is None:
|
||||
return block_table
|
||||
assert self.kernel_block_table is not None
|
||||
blocks_per_kv_block = self.kv_cache_spec.block_size // self.kernel_block_size
|
||||
rows, cols = block_table.shape
|
||||
out = self.kernel_block_table[ubatch_idx, :rows, : cols * blocks_per_kv_block]
|
||||
torch.add(
|
||||
self.kernel_block_offsets,
|
||||
block_table.unsqueeze(-1),
|
||||
alpha=self.kernel_block_stride or blocks_per_kv_block,
|
||||
out=out.unflatten(1, (cols, -1)),
|
||||
)
|
||||
return out
|
||||
|
||||
def map_kv_cache_to_kernel_blocks(self, kv_cache: torch.Tensor) -> torch.Tensor:
|
||||
"""Views a packed cache in manager blocks as kernel blocks."""
|
||||
assert self.kernel_block_size is not None
|
||||
spec = self.kv_cache_spec
|
||||
kernel_rows = spec.get_num_kernel_states(self.kernel_block_size)
|
||||
if kv_cache.shape[-2] == kernel_rows:
|
||||
return kv_cache
|
||||
# Token and pooled caches are written through manager-block slot
|
||||
# mappings, so only compressed caches (written from their block table)
|
||||
# can be mapped.
|
||||
if (
|
||||
spec.tokens_per_state <= 1
|
||||
or kv_cache.shape[1] != 1
|
||||
or kv_cache.stride(0) % (kernel_rows * kv_cache.stride(-2))
|
||||
):
|
||||
raise ValueError(
|
||||
f"{self.backend.get_name()} needs {self.kernel_block_size}-token "
|
||||
f"kernel blocks, which packed {spec.block_size}-token KV cache "
|
||||
f"blocks cannot be split into. Reduce --block-size to "
|
||||
f"{self.kernel_block_size} or set VLLM_KV_CACHE_LAYOUT to a "
|
||||
"layer-compact layout (e.g. LBNHC)."
|
||||
)
|
||||
# Kernel block j of block b is b * kernel_block_stride + j. The view
|
||||
# spans the other layers' pages between blocks (never addressing them),
|
||||
# which ``view`` cannot express.
|
||||
page_stride = kernel_rows * kv_cache.stride(-2)
|
||||
self.kernel_block_stride = kv_cache.stride(0) // page_stride
|
||||
num_pages = (kv_cache.shape[0] - 1) * self.kernel_block_stride + (
|
||||
kv_cache.shape[-2] // kernel_rows
|
||||
)
|
||||
return kv_cache.as_strided(
|
||||
(num_pages, *kv_cache.shape[1:-2], kernel_rows, kv_cache.shape[-1]),
|
||||
(page_stride, *kv_cache.stride()[1:]),
|
||||
)
|
||||
|
||||
def map_kv_cache_to_manager_blocks(
|
||||
self, kv_cache: torch.Tensor, num_blocks: int
|
||||
) -> torch.Tensor:
|
||||
"""Views a packed cache in kernel blocks as manager blocks."""
|
||||
assert self.kernel_block_stride
|
||||
spec = self.kv_cache_spec
|
||||
rows = spec.get_num_kernel_states(spec.block_size)
|
||||
return kv_cache.as_strided(
|
||||
(num_blocks, *kv_cache.shape[1:-2], rows, kv_cache.shape[-1]),
|
||||
(kv_cache.stride(0) * self.kernel_block_stride, *kv_cache.stride()[1:]),
|
||||
)
|
||||
|
||||
def get_metadata_builder(self, ubatch_id: int = 0) -> AttentionMetadataBuilder:
|
||||
assert len(self.metadata_builders) > ubatch_id
|
||||
@@ -327,17 +457,48 @@ class AttentionGroup:
|
||||
self.get_metadata_builder().update_draft_decode_metadata(metadata)
|
||||
|
||||
|
||||
def map_kv_caches_to_kernel_blocks(
|
||||
kv_caches: dict[str, torch.Tensor], attn_groups: Iterable[AttentionGroup]
|
||||
) -> dict[str, torch.Tensor]:
|
||||
"""The views to bind to layers. The manager views in ``kv_caches`` stay for
|
||||
KV connectors."""
|
||||
mapped = dict(kv_caches)
|
||||
for group in attn_groups:
|
||||
if group.kernel_block_size is not None:
|
||||
mapped.update(
|
||||
(name, group.map_kv_cache_to_kernel_blocks(kv_caches[name]))
|
||||
for name in group.layer_names
|
||||
if name in kv_caches
|
||||
)
|
||||
return mapped
|
||||
|
||||
|
||||
def customize_attention_spec(
|
||||
backend: type[AttentionBackend], spec: AttentionSpec
|
||||
) -> AttentionSpec:
|
||||
"""The backend's spec. Blocks larger than every kernel block size a backend
|
||||
takes are split into kernel blocks, which packed blocks must keep whole."""
|
||||
spec = backend.customize_spec(spec)
|
||||
sizes = backend.get_supported_kernel_block_sizes(spec)
|
||||
fixed_sizes = [s for s in sizes if isinstance(s, int)]
|
||||
if spec.block_stride_alignment is None and sizes and fixed_sizes == sizes:
|
||||
spec = replace(spec, block_stride_alignment=MultipleOf(max(fixed_sizes)))
|
||||
return spec
|
||||
|
||||
|
||||
def _block_size_is_supported(
|
||||
backends: list[type[AttentionBackend]], block_size: int
|
||||
backends: list[type[AttentionBackend]],
|
||||
block_size: int,
|
||||
specs: Sequence[KVCacheSpec | None],
|
||||
) -> bool:
|
||||
"""Check if the block size is supported by all backends.
|
||||
|
||||
An exact ``int`` declaration must match exactly; a ``MultipleOf``
|
||||
declaration accepts any multiple of its base.
|
||||
"""
|
||||
for backend in backends:
|
||||
for backend, spec in zip(backends, specs, strict=True):
|
||||
is_supported = False
|
||||
for supported_size in backend.get_supported_kernel_block_sizes():
|
||||
for supported_size in backend.get_supported_kernel_block_sizes(spec):
|
||||
if isinstance(supported_size, int):
|
||||
if block_size == supported_size:
|
||||
is_supported = True
|
||||
@@ -354,6 +515,7 @@ def _block_size_is_supported(
|
||||
def select_common_block_size(
|
||||
kv_manager_block_size: int,
|
||||
backends: list[type[AttentionBackend]],
|
||||
kv_cache_specs: Sequence[KVCacheSpec | None] | None = None,
|
||||
) -> int:
|
||||
"""Select a block size that is supported by all backends and is a factor of
|
||||
kv_manager_block_size.
|
||||
@@ -364,6 +526,7 @@ def select_common_block_size(
|
||||
Args:
|
||||
kv_manager_block_size: Block size of KV cache.
|
||||
backends: List of attention backend classes.
|
||||
kv_cache_specs: Per-backend cache specs for spec-specific constraints.
|
||||
|
||||
Returns:
|
||||
The selected block size.
|
||||
@@ -372,25 +535,27 @@ def select_common_block_size(
|
||||
ValueError: If no valid block size found.
|
||||
|
||||
"""
|
||||
if _block_size_is_supported(backends, kv_manager_block_size):
|
||||
specs = kv_cache_specs or [None] * len(backends)
|
||||
if _block_size_is_supported(backends, kv_manager_block_size, specs):
|
||||
return kv_manager_block_size
|
||||
|
||||
# MultipleOf constraints also accept the manager size if they accept a divisor.
|
||||
# Any remaining candidate must therefore be an explicit size from a backend.
|
||||
candidates = {
|
||||
size
|
||||
for backend in backends
|
||||
for size in backend.get_supported_kernel_block_sizes()
|
||||
for backend, spec in zip(backends, specs, strict=True)
|
||||
for size in backend.get_supported_kernel_block_sizes(spec)
|
||||
if isinstance(size, int) and kv_manager_block_size % size == 0
|
||||
}
|
||||
|
||||
for size in sorted(candidates, reverse=True):
|
||||
if _block_size_is_supported(backends, size):
|
||||
if _block_size_is_supported(backends, size, specs):
|
||||
return size
|
||||
raise ValueError(
|
||||
f"No common block size for {kv_manager_block_size} ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}" for b in backends
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes(s)}"
|
||||
for b, s in zip(backends, specs, strict=True)
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
@@ -450,9 +615,6 @@ def allocate_kv_cache(
|
||||
kernel_block_size = None
|
||||
if kernel_block_sizes is not None and group_id < len(kernel_block_sizes):
|
||||
kernel_block_size = kernel_block_sizes[group_id]
|
||||
if isinstance(spec, MLAAttentionSpec) and spec.storage_block_size is not None:
|
||||
kernel_block_size = spec.storage_block_size
|
||||
|
||||
views = create_kv_cache_views(
|
||||
buf,
|
||||
spec,
|
||||
@@ -497,24 +659,20 @@ def prepare_kernel_block_sizes(
|
||||
# This is an attention backend that supports virtual block splitting.
|
||||
kv_manager_block_size = kv_cache_group.kv_cache_spec.block_size
|
||||
group_backends = [g.backend for g in attn_groups[kv_cache_gid]]
|
||||
storage_block_size = (
|
||||
kv_cache_spec.storage_block_size
|
||||
if isinstance(kv_cache_spec, MLAAttentionSpec)
|
||||
else None
|
||||
)
|
||||
if storage_block_size is not None and _block_size_is_supported(
|
||||
group_backends, storage_block_size
|
||||
):
|
||||
# Storage-block specs (e.g. the GLM-5.3-Flash kpool indexer
|
||||
# cache) address the cache in pool pages, and every other
|
||||
# consumer (cache views, metadata builders, hisparse) already
|
||||
# uses storage_block_size as the kernel block. Fall back to the
|
||||
# backend vote when the group's backends do not accept it.
|
||||
selected_kernel_size = storage_block_size
|
||||
else:
|
||||
selected_kernel_size = select_common_block_size(
|
||||
kv_manager_block_size, group_backends
|
||||
group_specs = [g.kv_cache_spec for g in attn_groups[kv_cache_gid]]
|
||||
# Block-outermost packing puts other layers' pages between the
|
||||
# blocks, so they are never split group-wide.
|
||||
selected_kernel_size = (
|
||||
kv_manager_block_size
|
||||
if any(
|
||||
t.block_stride > t.layer_stride
|
||||
for t in kv_cache_config.kv_cache_tensors
|
||||
if not set(t.layers).isdisjoint(kv_cache_group.layer_names)
|
||||
)
|
||||
else select_common_block_size(
|
||||
kv_manager_block_size, group_backends, group_specs
|
||||
)
|
||||
)
|
||||
kernel_block_sizes.append(selected_kernel_size)
|
||||
elif isinstance(kv_cache_spec, MambaSpec):
|
||||
# This is likely Mamba or other non-attention cache, no splitting.
|
||||
@@ -656,6 +814,7 @@ def bind_kv_cache(
|
||||
runner_kv_caches: list[torch.Tensor],
|
||||
num_attn_module: int = 1,
|
||||
kv_cache_groups: Sequence[KVCacheGroupSpec] | None = None,
|
||||
layer_kv_caches: dict[str, torch.Tensor] | None = None,
|
||||
) -> None:
|
||||
"""Bind the allocated KV cache to both ModelRunner and forward context so
|
||||
that the KV cache can be used in the forward pass.
|
||||
@@ -674,6 +833,9 @@ def bind_kv_cache(
|
||||
runner_kv_caches: The kv_cache declared by ModelRunner.
|
||||
kv_cache_groups: The KV cache groups of the model, used to resolve
|
||||
layers that share a KV cache.
|
||||
layer_kv_caches: The views to bind to layers when they differ from
|
||||
kv_caches (e.g. kernel-block views). The runner keeps kv_caches,
|
||||
which are addressed by scheduler block IDs.
|
||||
|
||||
"""
|
||||
# Bind kv_caches to ModelRunner
|
||||
@@ -699,7 +861,7 @@ def bind_kv_cache(
|
||||
runner_kv_caches.append(kv_caches[layer_name])
|
||||
|
||||
bind_kv_cache_to_layers(
|
||||
kv_caches, forward_context, num_attn_module, kv_cache_groups
|
||||
layer_kv_caches or kv_caches, forward_context, num_attn_module, kv_cache_groups
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user