[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+21 -35
View File
@@ -4,6 +4,7 @@
from __future__ import annotations
import copy
import math
from collections import Counter
from collections.abc import Collection, Sequence
from dataclasses import dataclass, field, fields, replace
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.v1.attention.backend import MultipleOf
logger = init_logger(__name__)
@@ -168,13 +170,28 @@ class KVCacheSpec:
dcp_sharded: bool = field(default=False, kw_only=True)
"""Whether DCP shards this cache's token positions across ranks."""
block_stride_alignment: int | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers."""
block_stride_alignment: int | MultipleOf | None = field(default=None, kw_only=True)
"""Required byte alignment between physical blocks, including packed layers.
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
blocks of up to ``n`` tokens."""
def __post_init__(self):
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
alignment = self.block_stride_alignment
if isinstance(alignment, int) and alignment <= 0:
raise ValueError("block_stride_alignment must be positive")
def get_block_stride_alignment(self) -> int:
"""Bytes the stride between physical blocks must be a multiple of."""
from vllm.v1.attention.backend import MultipleOf
alignment = self.block_stride_alignment
if not isinstance(alignment, MultipleOf):
return alignment or 1
kernel_block_size = math.gcd(alignment.base, self.block_size)
if kernel_block_size == self.block_size:
return 1
return self.copy_with_new_block_size(kernel_block_size).page_size_bytes
@property
def prefix_cacheable(self) -> bool:
"""Whether this spec's group participates in prefix caching."""
@@ -674,8 +691,6 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version: str | None = None
cache_role: SparseCacheRole = SparseCacheRole.SPARSE
is_index_group_leader: bool = False
storage_block_size: int | None = None
"""Token width used to view storage when it differs from the kernel block."""
# Group capability enabled when any member flattens a non-causal query block
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
non_causal_multi_token_decode: bool = False
@@ -696,7 +711,6 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version_set = set(spec.model_version for spec in specs)
cache_role_set = {spec.cache_role for spec in specs}
index_group_leader_set = {spec.is_index_group_leader for spec in specs}
storage_block_size_set = set(spec.storage_block_size for spec in specs)
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
assert (
len(cache_dtype_str_set) == 1
@@ -704,12 +718,11 @@ class MLAAttentionSpec(FullAttentionSpec):
and len(model_version_set) == 1
and len(cache_role_set) == 1
and len(index_group_leader_set) == 1
and len(storage_block_size_set) == 1
and len(block_stride_alignment_set) == 1
), (
"All attention layers in the same KV cache group must use the same "
"quantization method, tokens per state, model version, cache role, "
"index-sharing role, storage block size and block stride alignment."
"index-sharing role, and block stride alignment."
)
merged_spec = cls(
block_size=specs[0].block_size,
@@ -727,7 +740,6 @@ class MLAAttentionSpec(FullAttentionSpec):
model_version=model_version_set.pop(),
cache_role=cache_role_set.pop(),
is_index_group_leader=index_group_leader_set.pop(),
storage_block_size=storage_block_size_set.pop(),
block_stride_alignment=block_stride_alignment_set.pop(),
non_causal_multi_token_decode=any(
spec.non_causal_multi_token_decode for spec in specs
@@ -1019,32 +1031,6 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
)
@dataclass(frozen=True, kw_only=True)
class KpoolTailSpec(SlidingWindowSpec):
"""One-block circular scratch cache for a kpool indexer's raw tail."""
def max_admission_blocks_per_request(
self, max_in_flight_tokens: int, max_model_len: int
) -> int:
return 1
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
return 1
def is_uniform_with_collection(
self, kv_cache_specs: dict[str, KVCacheSpec]
) -> bool:
return all(isinstance(spec, KpoolTailSpec) for spec in kv_cache_specs.values())
@property
def prefix_cacheable(self) -> bool:
return False
@property
def uses_slot_mapping(self) -> bool:
return False
@dataclass(frozen=True)
class MambaSpec(KVCacheSpec):
shapes: tuple[tuple[int, ...], ...]