[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
-20
View File
@@ -414,26 +414,6 @@ class CudaPlatformBase(Platform):
return valid_backends_priorities, invalid_reasons
@classmethod
def _get_indexer_block_alignment(cls, vllm_config: VllmConfig) -> int | None:
index_kpool = getattr(
vllm_config.model_config.hf_text_config, "index_kpool", None
)
if not index_kpool or index_kpool <= 1:
return None
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
# kpool paged-MQA indexer: the storage block (block_size /
# index_kpool) is virtually split into pool pages, so block_size
# must be a multiple of index_kpool times a legal pool page.
page = min(PAGED_MQA_PAGE_SIZES)
if cls.is_device_capability_family(120):
# On sm120 the DeepGEMM paged-MQA kernel only accepts block_kv
# 64 for the fp8 indexer cache, so align to the largest pool
# page here to make the page split land on 64 not the min 32.
page = max(PAGED_MQA_PAGE_SIZES)
return index_kpool * page
@classmethod
def get_attn_backend_cls(
cls,
+18 -21
View File
@@ -737,14 +737,14 @@ class Platform:
for candidate in candidates:
if all(b.supports_block_size(candidate) for b in backend_classes):
return candidate
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
raise ValueError(
"The attention backends share no supported KV cache block size ("
+ "; ".join(
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
for b in backend_classes
)
+ ")."
)
+ ")."
)
@classmethod
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
@@ -935,17 +935,6 @@ class Platform:
if cache_config.mamba_page_size_padded is not None:
cache_config.mamba_page_size_padded = shared_page
@classmethod
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
The CUDA kpool paged-MQA indexer virtually splits each storage block
into pool pages, so ``block_size`` must be a multiple of
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
platform override. Other platforms impose no extra constraint.
"""
return None
@classmethod
def _align_hybrid_block_size(
cls,
@@ -1077,6 +1066,14 @@ class Platform:
),
cache_config.block_size,
)
# Also a multiple of every backend's smallest kernel block.
backend_block_alignment = lcm(
*(
min(s.base if isinstance(s, MultipleOf) else s for s in sizes)
for b in cls._find_non_ssm_backends(vllm_config)
if (sizes := b.get_supported_kernel_block_sizes())
)
)
if model_config.use_mla:
# TRTLLM/FlashInfer MLA decode kernels require the physical
# number of kernel blocks to be aligned to 128 / kernel_block_size.
@@ -1090,9 +1087,9 @@ class Platform:
mamba_page_size,
kernel_block_alignment_size * attn_page_size_1_token,
)
indexer_align = cls._get_indexer_block_alignment(vllm_config)
if indexer_align:
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
attn_block_size = backend_block_alignment * cdiv(
attn_block_size, backend_block_alignment
)
if cache_config.block_size < attn_block_size:
cache_config.block_size = attn_block_size