[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -414,26 +414,6 @@ class CudaPlatformBase(Platform):
|
||||
|
||||
return valid_backends_priorities, invalid_reasons
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: VllmConfig) -> int | None:
|
||||
index_kpool = getattr(
|
||||
vllm_config.model_config.hf_text_config, "index_kpool", None
|
||||
)
|
||||
if not index_kpool or index_kpool <= 1:
|
||||
return None
|
||||
from vllm.utils.deep_gemm import PAGED_MQA_PAGE_SIZES
|
||||
|
||||
# kpool paged-MQA indexer: the storage block (block_size /
|
||||
# index_kpool) is virtually split into pool pages, so block_size
|
||||
# must be a multiple of index_kpool times a legal pool page.
|
||||
page = min(PAGED_MQA_PAGE_SIZES)
|
||||
if cls.is_device_capability_family(120):
|
||||
# On sm120 the DeepGEMM paged-MQA kernel only accepts block_kv
|
||||
# 64 for the fp8 indexer cache, so align to the largest pool
|
||||
# page here to make the page split land on 64 not the min 32.
|
||||
page = max(PAGED_MQA_PAGE_SIZES)
|
||||
return index_kpool * page
|
||||
|
||||
@classmethod
|
||||
def get_attn_backend_cls(
|
||||
cls,
|
||||
|
||||
+18
-21
@@ -737,14 +737,14 @@ class Platform:
|
||||
for candidate in candidates:
|
||||
if all(b.supports_block_size(candidate) for b in backend_classes):
|
||||
return candidate
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
raise ValueError(
|
||||
"The attention backends share no supported KV cache block size ("
|
||||
+ "; ".join(
|
||||
f"{b.get_name()}: {b.get_supported_kernel_block_sizes()}"
|
||||
for b in backend_classes
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
+ ")."
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def update_block_size_for_backend(cls, vllm_config: "VllmConfig") -> None:
|
||||
@@ -935,17 +935,6 @@ class Platform:
|
||||
if cache_config.mamba_page_size_padded is not None:
|
||||
cache_config.mamba_page_size_padded = shared_page
|
||||
|
||||
@classmethod
|
||||
def _get_indexer_block_alignment(cls, vllm_config: "VllmConfig") -> int | None:
|
||||
"""Extra ``block_size`` multiple a sparse indexer needs, else ``None``.
|
||||
|
||||
The CUDA kpool paged-MQA indexer virtually splits each storage block
|
||||
into pool pages, so ``block_size`` must be a multiple of
|
||||
``index_kpool * min(PAGED_MQA_PAGE_SIZES)`` — implemented in the CUDA
|
||||
platform override. Other platforms impose no extra constraint.
|
||||
"""
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _align_hybrid_block_size(
|
||||
cls,
|
||||
@@ -1077,6 +1066,14 @@ class Platform:
|
||||
),
|
||||
cache_config.block_size,
|
||||
)
|
||||
# Also a multiple of every backend's smallest kernel block.
|
||||
backend_block_alignment = lcm(
|
||||
*(
|
||||
min(s.base if isinstance(s, MultipleOf) else s for s in sizes)
|
||||
for b in cls._find_non_ssm_backends(vllm_config)
|
||||
if (sizes := b.get_supported_kernel_block_sizes())
|
||||
)
|
||||
)
|
||||
if model_config.use_mla:
|
||||
# TRTLLM/FlashInfer MLA decode kernels require the physical
|
||||
# number of kernel blocks to be aligned to 128 / kernel_block_size.
|
||||
@@ -1090,9 +1087,9 @@ class Platform:
|
||||
mamba_page_size,
|
||||
kernel_block_alignment_size * attn_page_size_1_token,
|
||||
)
|
||||
indexer_align = cls._get_indexer_block_alignment(vllm_config)
|
||||
if indexer_align:
|
||||
attn_block_size = indexer_align * cdiv(attn_block_size, indexer_align)
|
||||
attn_block_size = backend_block_alignment * cdiv(
|
||||
attn_block_size, backend_block_alignment
|
||||
)
|
||||
|
||||
if cache_config.block_size < attn_block_size:
|
||||
cache_config.block_size = attn_block_size
|
||||
|
||||
Reference in New Issue
Block a user