[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -20,6 +20,7 @@ from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.attention.backend import AttentionBackend
|
||||
from vllm.v1.kv_cache_interface import AttentionSpec, KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput, ModelRunnerOutput
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -351,6 +352,35 @@ def get_current_attn_backends(
|
||||
]
|
||||
|
||||
|
||||
def get_current_attn_backends_and_specs(
|
||||
vllm_config: VllmConfig,
|
||||
kv_cache_config: KVCacheConfig,
|
||||
fallback: list[type[AttentionBackend]],
|
||||
) -> tuple[list[type[AttentionBackend]], list[AttentionSpec | None]]:
|
||||
"""Distinct (backend, spec) pairs of the transfer layers, else ``fallback``.
|
||||
|
||||
Compressed caches are skipped: their kernel pages never split transferred
|
||||
blocks.
|
||||
"""
|
||||
pairs: dict[tuple[type[AttentionBackend], AttentionSpec | None], None] = {}
|
||||
layer_type = cast(type[Any], AttentionLayerBase)
|
||||
for group in kv_cache_config.transfer_groups:
|
||||
specs = getattr(group.kv_cache_spec, "kv_cache_specs", {})
|
||||
layers = get_layers_from_vllm_config(vllm_config, layer_type, group.layer_names)
|
||||
for name, layer in layers.items():
|
||||
spec = specs.get(name, group.kv_cache_spec)
|
||||
if isinstance(spec, AttentionSpec) and spec.tokens_per_state > 1:
|
||||
layout = vllm_config.cache_config.get_resolved_kv_cache_layout()
|
||||
assert layout.is_block_outermost, "Compressed caches must be packed"
|
||||
continue
|
||||
if not isinstance(spec, AttentionSpec):
|
||||
spec = None
|
||||
pairs[layer.get_attn_backend(), spec] = None
|
||||
if not pairs:
|
||||
return fallback, [None] * len(fallback)
|
||||
return [backend for backend, _ in pairs], [spec for _, spec in pairs]
|
||||
|
||||
|
||||
def get_current_attn_backend(
|
||||
vllm_config: VllmConfig, layer_names: list[str] | None = None
|
||||
) -> type[AttentionBackend]:
|
||||
|
||||
Reference in New Issue
Block a user