[KV Cache] GLM-5.3-Flash: use the generic packed KV layout; CircularBufferSpec tail with speculative ring slots (#57169)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Signed-off-by: Dakai An <dakaian108@gmail.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
co-authored by
Yifan Qiao
OpenAI Codex
OpenAI Codex
Lucas Wilkinson
Claude Opus 5.5
Lucas Wilkinson
parent
3766a67a32
commit
02e55d2af3
@@ -4,6 +4,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import math
|
||||
from collections import Counter
|
||||
from collections.abc import Collection, Sequence
|
||||
from dataclasses import dataclass, field, fields, replace
|
||||
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -168,13 +170,28 @@ class KVCacheSpec:
|
||||
dcp_sharded: bool = field(default=False, kw_only=True)
|
||||
"""Whether DCP shards this cache's token positions across ranks."""
|
||||
|
||||
block_stride_alignment: int | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers."""
|
||||
block_stride_alignment: int | MultipleOf | None = field(default=None, kw_only=True)
|
||||
"""Required byte alignment between physical blocks, including packed layers.
|
||||
``MultipleOf(n)``: whole kernel blocks, when blocks are split into kernel
|
||||
blocks of up to ``n`` tokens."""
|
||||
|
||||
def __post_init__(self):
|
||||
if self.block_stride_alignment is not None and self.block_stride_alignment <= 0:
|
||||
alignment = self.block_stride_alignment
|
||||
if isinstance(alignment, int) and alignment <= 0:
|
||||
raise ValueError("block_stride_alignment must be positive")
|
||||
|
||||
def get_block_stride_alignment(self) -> int:
|
||||
"""Bytes the stride between physical blocks must be a multiple of."""
|
||||
from vllm.v1.attention.backend import MultipleOf
|
||||
|
||||
alignment = self.block_stride_alignment
|
||||
if not isinstance(alignment, MultipleOf):
|
||||
return alignment or 1
|
||||
kernel_block_size = math.gcd(alignment.base, self.block_size)
|
||||
if kernel_block_size == self.block_size:
|
||||
return 1
|
||||
return self.copy_with_new_block_size(kernel_block_size).page_size_bytes
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
"""Whether this spec's group participates in prefix caching."""
|
||||
@@ -674,8 +691,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version: str | None = None
|
||||
cache_role: SparseCacheRole = SparseCacheRole.SPARSE
|
||||
is_index_group_leader: bool = False
|
||||
storage_block_size: int | None = None
|
||||
"""Token width used to view storage when it differs from the kernel block."""
|
||||
# Group capability enabled when any member flattens a non-causal query block
|
||||
# into decode rows. Runtime metadata still selects causal vs. non-causal mode.
|
||||
non_causal_multi_token_decode: bool = False
|
||||
@@ -696,7 +711,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version_set = set(spec.model_version for spec in specs)
|
||||
cache_role_set = {spec.cache_role for spec in specs}
|
||||
index_group_leader_set = {spec.is_index_group_leader for spec in specs}
|
||||
storage_block_size_set = set(spec.storage_block_size for spec in specs)
|
||||
block_stride_alignment_set = {spec.block_stride_alignment for spec in specs}
|
||||
assert (
|
||||
len(cache_dtype_str_set) == 1
|
||||
@@ -704,12 +718,11 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
and len(model_version_set) == 1
|
||||
and len(cache_role_set) == 1
|
||||
and len(index_group_leader_set) == 1
|
||||
and len(storage_block_size_set) == 1
|
||||
and len(block_stride_alignment_set) == 1
|
||||
), (
|
||||
"All attention layers in the same KV cache group must use the same "
|
||||
"quantization method, tokens per state, model version, cache role, "
|
||||
"index-sharing role, storage block size and block stride alignment."
|
||||
"index-sharing role, and block stride alignment."
|
||||
)
|
||||
merged_spec = cls(
|
||||
block_size=specs[0].block_size,
|
||||
@@ -727,7 +740,6 @@ class MLAAttentionSpec(FullAttentionSpec):
|
||||
model_version=model_version_set.pop(),
|
||||
cache_role=cache_role_set.pop(),
|
||||
is_index_group_leader=index_group_leader_set.pop(),
|
||||
storage_block_size=storage_block_size_set.pop(),
|
||||
block_stride_alignment=block_stride_alignment_set.pop(),
|
||||
non_causal_multi_token_decode=any(
|
||||
spec.non_causal_multi_token_decode for spec in specs
|
||||
@@ -1019,32 +1031,6 @@ class SlidingWindowMLASpec(SlidingWindowSpec):
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True, kw_only=True)
|
||||
class KpoolTailSpec(SlidingWindowSpec):
|
||||
"""One-block circular scratch cache for a kpool indexer's raw tail."""
|
||||
|
||||
def max_admission_blocks_per_request(
|
||||
self, max_in_flight_tokens: int, max_model_len: int
|
||||
) -> int:
|
||||
return 1
|
||||
|
||||
def max_num_blocks_per_req(self, vllm_config: VllmConfig, max_len: int) -> int:
|
||||
return 1
|
||||
|
||||
def is_uniform_with_collection(
|
||||
self, kv_cache_specs: dict[str, KVCacheSpec]
|
||||
) -> bool:
|
||||
return all(isinstance(spec, KpoolTailSpec) for spec in kv_cache_specs.values())
|
||||
|
||||
@property
|
||||
def prefix_cacheable(self) -> bool:
|
||||
return False
|
||||
|
||||
@property
|
||||
def uses_slot_mapping(self) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MambaSpec(KVCacheSpec):
|
||||
shapes: tuple[tuple[int, ...], ...]
|
||||
|
||||
Reference in New Issue
Block a user