[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Signed-off-by: Dakai An <dakaian108@gmail.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Dakai An
2026-10-09 15:48:37 -04:00
committed by GitHub
co-authored by Yifan Qiao OpenAI Codex OpenAI Codex Lucas Wilkinson Claude Opus 5.5 Lucas Wilkinson
parent 3766a67a32
commit 02e55d2af3
83 changed files with 1280 additions and 1986 deletions
+13 -11
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence
from contextlib import AbstractContextManager, nullcontext
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, cast
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
add_kv_sharing_layers_to_kv_cache_groups,
allocate_kv_cache,
bind_kv_cache_to_layers,
customize_attention_spec,
map_kv_caches_to_kernel_blocks,
prepare_kernel_block_sizes,
)
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
# Skip modules that don't need KV cache (eg encoder-only attention)
if spec := attn_module.get_kv_cache_spec(vllm_config):
if isinstance(spec, AttentionSpec):
spec = attn_module.get_attn_backend().customize_spec(spec)
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
kv_cache_spec[layer_name] = spec
if vllm_config.attention_config.hisparse_config is not None:
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
@@ -382,6 +384,7 @@ def init_kv_cache(
kv_cache_allocation_context: AbstractContextManager | None = None,
*,
block_tables: "BlockTables | None" = None,
attn_groups: Iterable[AttentionGroup] = (),
) -> dict[str, Any]:
allocation_context = kv_cache_allocation_context or nullcontext()
with allocation_context:
@@ -409,9 +412,10 @@ def init_kv_cache(
num_attn_module = (
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
)
bindable_caches = {
name: cache for name, cache in kv_caches.items() if name in forward_context
}
bindable_caches = map_kv_caches_to_kernel_blocks(
{name: cache for name, cache in kv_caches.items() if name in forward_context},
attn_groups,
)
bind_kv_cache_to_layers(
bindable_caches,
forward_context,
@@ -536,8 +540,8 @@ def build_attn_metadata(
cached_metadata[reuse_key], block_table, slot_mapping
)
elif for_cudagraph_capture:
metadata = attn_metadata_builder.build_for_cudagraph_capture(
common_attn_metadata
metadata = attn_group.build_metadata_for_cudagraph_capture(
common_attn_metadata, ubatch_idx
)
else:
attn_metadata_extra_kwargs = (
@@ -548,10 +552,8 @@ def build_attn_metadata(
if model_specific_attn_metadata is not None
else {}
)
metadata = attn_metadata_builder.build(
common_prefix_len=0,
common_attn_metadata=common_attn_metadata,
**attn_metadata_extra_kwargs,
metadata = attn_group.build_metadata(
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
)
if reuse_key is not None and reuse_key not in cached_metadata:
cached_metadata[reuse_key] = metadata