[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -47,7 +47,6 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
KVCacheTensor,
|
||||
@@ -145,13 +144,12 @@ def _make_scratch_kv_cache_config(
|
||||
register_all_kvcache_specs(vllm_config=None)
|
||||
fa_config = _make_kv_cache_config(num_blocks, num_groups=1)
|
||||
scratch_layers = ["layer_scratch"]
|
||||
scratch_spec = KpoolTailSpec(
|
||||
scratch_spec = CircularBufferSpec(
|
||||
block_size=scratch_block_size,
|
||||
num_kv_heads=2,
|
||||
head_size=HEAD_SIZE,
|
||||
head_size_v=0,
|
||||
dtype=DTYPE,
|
||||
sliding_window=scratch_block_size,
|
||||
)
|
||||
assert not scratch_spec.prefix_cacheable
|
||||
scratch_bytes = scratch_spec.page_size_bytes * num_blocks
|
||||
|
||||
Reference in New Issue
Block a user