[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+18 -21
View File
@@ -737,14 +737,14 @@ class Platform:
for candidate in candidates:
if all(b.supports_block_size(candidate) for b in backend_classes):
return candidate
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
)
+ ")."
)
+ ")."
)
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
@@ -935,17 +935,6 @@ class Platform:
if cache_config.mamba_page_size_padded is not None:
cache_config.mamba_page_size_padded = shared_page
@classmethod
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
The CUDA kpool paged-MQA indexer virtually splits each storage block
into pool pages, so ``block_size`` must be a multiple of
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
platform override. Other platforms impose no extra constraint.
"""
return None
@classmethod
def _align_hybrid_block_size(
cls,
@@ -1077,6 +1066,14 @@ class Platform:
),
cache_config.block_size,
)
# Also a multiple of every backend's smallest kernel block.
backend_block_alignment = lcm(
*(
min(s.base if isinstance(s, MultipleOf) else s for s in sizes)
for b in cls._find_non_ssm_backends(vllm_config)
if (sizes := b.get_supported_kernel_block_sizes())
)
)
if model_config.use_mla:
# TRTLLM/FlashInfer MLA decode kernels require the physical
# number of kernel blocks to be aligned to 128 / kernel_block_size.
@@ -1090,9 +1087,9 @@ class Platform:
mamba_page_size,
kernel_block_alignment_size * attn_page_size_1_token,
)
indexer_align = cls._get_indexer_block_alignment(vllm_config)
if indexer_align:
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
attn_block_size = backend_block_alignment * cdiv(
attn_block_size, backend_block_alignment
)
if cache_config.block_size < attn_block_size:
cache_config.block_size = attn_block_size