[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections.abc import Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
@@ -40,6 +40,8 @@ from vllm.v1.worker.utils import (
|
||||
add_kv_sharing_layers_to_kv_cache_groups,
|
||||
allocate_kv_cache,
|
||||
bind_kv_cache_to_layers,
|
||||
customize_attention_spec,
|
||||
map_kv_caches_to_kernel_blocks,
|
||||
prepare_kernel_block_sizes,
|
||||
)
|
||||
|
||||
@@ -141,7 +143,7 @@ def get_kv_cache_spec(vllm_config: VllmConfig) -> dict[str, KVCacheSpec]:
|
||||
# Skip modules that don't need KV cache (eg encoder-only attention)
|
||||
if spec := attn_module.get_kv_cache_spec(vllm_config):
|
||||
if isinstance(spec, AttentionSpec):
|
||||
spec = attn_module.get_attn_backend().customize_spec(spec)
|
||||
spec = customize_attention_spec(attn_module.get_attn_backend(), spec)
|
||||
kv_cache_spec[layer_name] = spec
|
||||
if vllm_config.attention_config.hisparse_config is not None:
|
||||
kv_cache_spec = resolve_hisparse_specs(vllm_config, kv_cache_spec, attn_layers)
|
||||
@@ -382,6 +384,7 @@ def init_kv_cache(
|
||||
kv_cache_allocation_context: AbstractContextManager | None = None,
|
||||
*,
|
||||
block_tables: "BlockTables | None" = None,
|
||||
attn_groups: Iterable[AttentionGroup] = (),
|
||||
) -> dict[str, Any]:
|
||||
allocation_context = kv_cache_allocation_context or nullcontext()
|
||||
with allocation_context:
|
||||
@@ -409,9 +412,10 @@ def init_kv_cache(
|
||||
num_attn_module = (
|
||||
2 if vllm_config.model_config.hf_config.model_type == "longcat_flash" else 1
|
||||
)
|
||||
bindable_caches = {
|
||||
name: cache for name, cache in kv_caches.items() if name in forward_context
|
||||
}
|
||||
bindable_caches = map_kv_caches_to_kernel_blocks(
|
||||
{name: cache for name, cache in kv_caches.items() if name in forward_context},
|
||||
attn_groups,
|
||||
)
|
||||
bind_kv_cache_to_layers(
|
||||
bindable_caches,
|
||||
forward_context,
|
||||
@@ -536,8 +540,8 @@ def build_attn_metadata(
|
||||
cached_metadata[reuse_key], block_table, slot_mapping
|
||||
)
|
||||
elif for_cudagraph_capture:
|
||||
metadata = attn_metadata_builder.build_for_cudagraph_capture(
|
||||
common_attn_metadata
|
||||
metadata = attn_group.build_metadata_for_cudagraph_capture(
|
||||
common_attn_metadata, ubatch_idx
|
||||
)
|
||||
else:
|
||||
attn_metadata_extra_kwargs = (
|
||||
@@ -548,10 +552,8 @@ def build_attn_metadata(
|
||||
if model_specific_attn_metadata is not None
|
||||
else {}
|
||||
)
|
||||
metadata = attn_metadata_builder.build(
|
||||
common_prefix_len=0,
|
||||
common_attn_metadata=common_attn_metadata,
|
||||
**attn_metadata_extra_kwargs,
|
||||
metadata = attn_group.build_metadata(
|
||||
common_attn_metadata, ubatch_idx, **attn_metadata_extra_kwargs
|
||||
)
|
||||
if reuse_key is not None and reuse_key not in cached_metadata:
|
||||
cached_metadata[reuse_key] = metadata
|
||||
|
||||
Reference in New Issue
Block a user