[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
+18
-21
@@ -737,14 +737,14 @@ class Platform:
|
||||
for candidate in candidates:
|
||||
if all(b.supports_block_size(candidate) for b in backend_classes):
|
||||
return candidate
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -935,17 +935,6 @@ class Platform:
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
cache_config.mamba_page_size_padded = shared_page
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
|
||||
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
|
||||
|
||||
The CUDA kpool paged-MQA indexer virtually splits each storage block
|
||||
into pool pages, so ``block_size`` must be a multiple of
|
||||
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
|
||||
platform override. Other platforms impose no extra constraint.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
@@ -1077,6 +1066,14 @@ class Platform:
|
||||
),
|
||||
cache_config.block_size,
|
||||
)
|
||||
# Also a multiple of every backend's smallest kernel block.
|
||||
backend_block_alignment = lcm(
|
||||
*(
|
||||
min(s.base if isinstance(s, MultipleOf) else s for s in sizes)
|
||||
for b in cls._find_non_ssm_backends(vllm_config)
|
||||
if (sizes := b.get_supported_kernel_block_sizes())
|
||||
)
|
||||
)
|
||||
if model_config.use_mla:
|
||||
# TRTLLM/FlashInfer MLA decode kernels require the physical
|
||||
# number of kernel blocks to be aligned to 128 / kernel_block_size.
|
||||
@@ -1090,9 +1087,9 @@ class Platform:
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
attn_block_size = backend_block_alignment * cdiv(
|
||||
attn_block_size, backend_block_alignment
|
||||
)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
|
||||
Reference in New Issue
Block a user