diff --git a/vllm/envs.py b/vllm/envs.py index 42e72f3652..baa45b8fbd 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -458,6 +458,16 @@ if TYPE_CHECKING: # for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only; # the worker sees the same block tables either way. VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True + # v2.5 side-cache layout (Flash-Next PLE + QSA ring): stop charging each + # tiny side cache a whole packed pool block per request. The TP-replicated + # PLE short-conv state group is folded into a GDN group that has a spare + # page on every PP stage, and the one-block-per-request QSA ring group packs + # several requests' rings into one pool block. Strict "0"/"1", default ON; + # "0" restores the old one-group-each layout for A/B. Read in + # vllm/v1/core/kv_cache_utils.py (get_kv_cache_configs). AOT: yes, hashed -- + # it changes the ring views' block stride and which block table the PLE + # state is read through, so ON and OFF must not share a compile cache. + VLLM_E2_SIDE_CACHE_LAYOUT: bool = True # Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at # Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes # nothing itself; it only permits a boot whose identity IS different @@ -2694,6 +2704,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch( "VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True ), + # Side-cache layout (v2.5; PLE state folded into a GDN group + QSA ring slot + # packing), default ON. Strict "0"/"1"; read once per get_kv_cache_configs call. + "VLLM_E2_SIDE_CACHE_LAYOUT": lambda: _e2_strict_switch( + "VLLM_E2_SIDE_CACHE_LAYOUT", default=True + ), # Seconds between `topk-exact-counters` lines from each GPU worker # (0, the default, disables the reader entirely). The line reports the # medium-path exact-restart counters; see diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index da1e57e0e7..a53d075fb1 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -13,6 +13,7 @@ from vllm.v1.core.kv_cache_utils import ( KVCacheBlock, ) from vllm.v1.core.single_type_kv_cache_manager import ( + CircularBufferManager, CrossAttentionManager, MambaManager, SingleTypeKVCacheManager, @@ -147,6 +148,17 @@ class KVCacheCoordinator(ABC): ) for i, kv_cache_group in enumerate(self.kv_cache_config.kv_cache_groups) ) + # v2.5 side-cache layout: a slot-packed ring group shares each pool + # block between several requests (kv_cache_utils._choose_ring_slots). + for kv_cache_group, manager in zip( + self.kv_cache_config.kv_cache_groups, self.single_type_managers + ): + if kv_cache_group.slots_per_block > 1: + assert type(manager) is CircularBufferManager, ( + f"slot packing is only defined for the QSA ring, got " + f"{type(manager).__name__}" + ) + manager.enable_slot_packing(kv_cache_group.slots_per_block) # Match Mamba checkpoints to Eagle's attention replay boundary. if use_eagle: for manager in self.single_type_managers: diff --git a/vllm/v1/core/kv_cache_utils.py b/vllm/v1/core/kv_cache_utils.py index 7da377146d..9696f7f33e 100644 --- a/vllm/v1/core/kv_cache_utils.py +++ b/vllm/v1/core/kv_cache_utils.py @@ -1121,7 +1121,19 @@ def get_max_concurrency_for_kv_cache_config( """ num_blocks_per_request = 0 host_blocks_per_request = 0 + # v2.5 side-cache layout: a slot-packed ring group shares each pool block + # between `slots_per_block` requests, so it costs cdiv(max_num_seqs, slots) + # blocks for the whole pool instead of one block per request. + shared_blocks = 0 + max_num_seqs = getattr( + getattr(vllm_config, "scheduler_config", None), "max_num_seqs", 1 + ) + if not isinstance(max_num_seqs, int) or max_num_seqs < 1: + max_num_seqs = 1 for group in kv_cache_config.kv_cache_groups: + if group.slots_per_block > 1 and not group.host_resident: + shared_blocks += cdiv(max_num_seqs, group.slots_per_block) + continue required = cdiv( group.kv_cache_spec.max_memory_usage_bytes(vllm_config), group.kv_cache_spec.page_size_bytes, @@ -1130,7 +1142,9 @@ def get_max_concurrency_for_kv_cache_config( host_blocks_per_request += required else: num_blocks_per_request += required - limits = [kv_cache_config.num_blocks / num_blocks_per_request] + limits = [ + max(kv_cache_config.num_blocks - shared_blocks, 0) / num_blocks_per_request + ] if host_blocks_per_request: assert kv_cache_config.hisparse_host_num_blocks is not None limits.append( @@ -1817,14 +1831,36 @@ def get_kv_cache_config_from_groups( elif group.layer_names: layers_by_spec[group_spec].extend(group.layer_names) + # v2.5 side-cache layout: a slot-packed group views every pool block as + # `slots` equal slots, so its block stride is one slot and its view has + # num_blocks * slots blocks. Pages of its layers must fit in one slot. + slots = group.slots_per_block + group_block_stride = interleaved_block_stride + group_num_blocks = num_blocks + if slots > 1: + assert layout.is_block_outermost, ( + "Slot-packed KV cache groups need a block-outermost layout" + ) + assert bytes_per_block % slots == 0, (bytes_per_block, slots) + group_block_stride = bytes_per_block // slots + group_num_blocks = num_blocks * slots + group_bytes = sum( + spec.page_size_bytes * len(names) + for spec, names in layers_by_spec.items() + ) + assert group_bytes <= group_block_stride, ( + f"slot-packed group needs {group_bytes} B per slot, slot is " + f"{group_block_stride} B" + ) + byte_offset = 0 for spec, layer_names in layers_by_spec.items(): layer_stride, block_stride, _, _, _ = compute_layout_strides( spec, - num_blocks, + group_num_blocks, len(layer_names), layout, - fixed_strides=(None, interleaved_block_stride, None, None, None), + fixed_strides=(None, group_block_stride, None, None, None), ) offset = ( byte_offset @@ -1838,6 +1874,7 @@ def get_kv_cache_config_from_groups( layer_stride=layer_stride, block_stride=block_stride, offset=offset, + slots_per_block=slots, ) ) byte_offset += len(layer_names) * spec.page_size_bytes @@ -2705,6 +2742,289 @@ def _project_kv_cache_groups_to_worker( return projected_groups +# --------------------------------------------------------------------------- +# v2.5 side-cache layout (VLLM_E2_SIDE_CACHE_LAYOUT, default ON). +# +# The packed block-outermost layout strides every pool block by the widest +# group, and every group draws whole blocks from the one pool. Flash-Next has +# two side caches whose real state is tiny next to that stride (12.9 MB on the +# binding PP stage): the TP-replicated PLE short-conv state (one layer, 2 + K +# blocks per request in align mode, a 1.6 MB padded page) and the QSA raw-key +# ring (one block per request, 16 KB for all of a stage's layers). Together +# they hold 6 of the 103 blocks a 262K request needs. Two layout-only changes +# remove that cost without changing any value a kernel reads: +# +# 1. PLE fold. The replicated Mamba group joins a sharded Mamba group with the +# same scheduling parameters whose block has a spare page on every worker +# (Flash-Next: a GDN group holds 6 of PP0's 7 pages), so the PLE state rides +# in that group's blocks under the block table it would have had on its +# own. No worker's stride changes, so the pool block count does not change. +# 2. Ring slot packing. The ring group splits every pool block into S equal +# slots and the ring manager hands out slots; a pool block goes back to the +# pool when its last slot is released. The ring then costs cdiv(requests, S) +# blocks in all instead of one per request. +# +# Off (=0) restores one group each, block for block. +# --------------------------------------------------------------------------- + +# Slot stride alignment for packed ring views (bytes) and the largest S tried. +_RING_SLOT_ALIGN = 256 +_RING_MAX_SLOTS = 1024 + + +def _side_cache_layout_enabled(vllm_config: VllmConfig) -> bool: + """Whether the v2.5 side-cache layout may apply to this engine. + + KV connectors (NIXL, offload, P/D) address groups and blocks themselves + (the NIXL worker requires exactly one single-layer replicated group), and + HiSparse has its own pool layout, so those engines keep the old layout. + """ + if not envs.VLLM_E2_SIDE_CACHE_LAYOUT: + return False + if getattr(vllm_config, "kv_transfer_config", None) is not None: + return False + attention_config = getattr(vllm_config, "attention_config", None) + if getattr(attention_config, "hisparse_config", None) is not None: + return False + parallel_config = getattr(vllm_config, "parallel_config", None) + if getattr(parallel_config, "decode_context_parallel_size", 1) not in (1, None): + return False + try: + layout = vllm_config.cache_config.get_resolved_kv_cache_layout() + except ValueError: + # Unresolved layout: only reachable from callers that bypass the + # engine core; nothing to pack without a block-outermost layout. + return False + return bool(layout.is_block_outermost) + + +def _group_layer_specs(group: KVCacheGroupSpec) -> list[KVCacheSpec]: + spec = group.kv_cache_spec + if isinstance(spec, UniformTypeKVCacheSpecs): + return [spec.kv_cache_specs[name] for name in group.layer_names] + return [spec] * len(group.layer_names) + + +def _mamba_schedule_key(spec: MambaSpec) -> tuple: + """Everything the scheduler's MambaManager and the worker's align-mode + bookkeeping read from a group's spec. Two Mamba groups with equal keys get + identical block tables for every request.""" + return ( + spec.block_size, + spec.page_size_bytes, + spec.num_speculative_blocks, + spec.num_prefill_checkpoint_blocks, + spec.prefill_checkpoint_alignment, + spec.mamba_cache_mode, + ) + + +def _worker_strides( + groups: list[KVCacheGroupSpec], worker_specs: list[dict[str, KVCacheSpec]] +) -> list[int]: + """Pool block bytes of every worker for this global grouping.""" + strides = [] + for spec in worker_specs: + projected = _project_kv_cache_groups_to_worker(groups, spec) + if not any(g.layer_names for g in projected): + strides.append(0) + continue + strides.append(_get_kv_cache_bytes_per_block(projected)) + return strides + + +def _fold_replicated_mamba_groups( + groups: list[KVCacheGroupSpec], + worker_specs: list[dict[str, KVCacheSpec]], +) -> tuple[list[KVCacheGroupSpec], list[str]]: + """Fold each TP-replicated Mamba group into a compatible sharded one. + + A target holds only sharded Mamba layers with the same scheduling key and + EAGLE flag, and the merged group must not widen the block of any worker + (checked on every worker's projection, so PP stages are covered). Among + valid targets the lowest index wins. Returns the new group list and one + log note per replicated group. + """ + groups = list(groups) + notes: list[str] = [] + base_strides = _worker_strides(groups, worker_specs) + index = 0 + while index < len(groups): + source = groups[index] + layer_specs = _group_layer_specs(source) + if not ( + layer_specs + and all( + isinstance(spec, MambaSpec) and spec.tp_replicated + for spec in layer_specs + ) + and len({_mamba_schedule_key(spec) for spec in layer_specs}) == 1 + ): + index += 1 + continue + key = _mamba_schedule_key(cast(MambaSpec, layer_specs[0])) + chosen = None + for target_index, target in enumerate(groups): + if target_index == index or target.is_eagle_group != source.is_eagle_group: + continue + target_specs = _group_layer_specs(target) + if not target_specs or not all( + isinstance(spec, MambaSpec) + and not spec.tp_replicated + and _mamba_schedule_key(spec) == key + for spec in target_specs + ): + continue + merged_specs: dict[str, KVCacheSpec] = dict( + zip(target.layer_names, target_specs) + ) + merged_specs.update(zip(source.layer_names, layer_specs)) + # Built directly: is_uniform_type rejects mixed tp_replicated on + # purpose (NIXL), and connectors never reach this path. + merged = KVCacheGroupSpec( + target.layer_names + source.layer_names, + UniformTypeKVCacheSpecs( + block_size=target_specs[0].block_size, + kv_cache_specs=merged_specs, + ), + is_eagle_group=target.is_eagle_group, + ) + trial = [ + merged if i == target_index else g + for i, g in enumerate(groups) + if i != index + ] + strides = _worker_strides(trial, worker_specs) + if all(new <= old for new, old in zip(strides, base_strides)): + chosen = (target_index, merged, trial) + break + if chosen is None: + notes.append( + f"replicated Mamba group {index} ({len(source.layer_names)} " + "layer(s)) kept separate: no sharded group has a spare page on " + "every worker" + ) + index += 1 + continue + target_index, merged, trial = chosen + notes.append( + f"folded replicated Mamba group {index} " + f"({', '.join(source.layer_names)}) into Mamba group " + f"{target_index} (now {len(merged.layer_names)} layers); per-worker " + f"block bytes unchanged {sorted(set(base_strides))}" + ) + groups = trial + # The group after the folded one now sits at `index`. + return groups, notes + + +def _is_ring_group(group: KVCacheGroupSpec) -> bool: + spec = group.kv_cache_spec + layer_specs = ( + list(spec.kv_cache_specs.values()) + if isinstance(spec, UniformTypeKVCacheSpecs) + else [spec] + ) + return bool(layer_specs) and all( + type(layer_spec) is CircularBufferSpec for layer_spec in layer_specs + ) + + +def _choose_ring_slots( + projected_groups_per_worker: list[list[KVCacheGroupSpec]], +) -> dict[int, int]: + """Slots per pool block for every ring group (group index -> S >= 2). + + S must cut every worker's block stride into equal slots that are + `_RING_SLOT_ALIGN`-aligned and hold that worker's ring pages; the largest + such S up to `_RING_MAX_SLOTS` is taken. Block ids are global, so S is the + same on every worker. A ring group with no valid S keeps one pool block + per request. + """ + chosen: dict[int, int] = {} + if not projected_groups_per_worker or not projected_groups_per_worker[0]: + return chosen + num_groups = len(projected_groups_per_worker[0]) + strides = [ + _get_kv_cache_bytes_per_block(groups) + if any(g.layer_names for g in groups) + else 0 + for groups in projected_groups_per_worker + ] + for group_index in range(num_groups): + if not _is_ring_group(projected_groups_per_worker[0][group_index]): + continue + needs = [] + for groups, stride in zip(projected_groups_per_worker, strides): + group = groups[group_index] + if not group.layer_names: + continue + ring_bytes = sum(spec.page_size_bytes for spec in _group_layer_specs(group)) + needs.append((stride, ring_bytes)) + if not needs: + continue + for slots in range(_RING_MAX_SLOTS, 1, -1): + if all( + stride % slots == 0 + and (stride // slots) % _RING_SLOT_ALIGN == 0 + and stride // slots >= ring_bytes + for stride, ring_bytes in needs + ): + chosen[group_index] = slots + break + return chosen + + +def _apply_side_cache_layout( + vllm_config: VllmConfig, + global_kv_cache_groups: list[KVCacheGroupSpec], + kv_cache_specs: list[dict[str, KVCacheSpec]], +) -> tuple[list[KVCacheGroupSpec], list[list[KVCacheGroupSpec]]]: + """Fold side Mamba groups, project per worker, then pick ring slots. + + Returns the (possibly folded) global groups and every worker's projection, + with `slots_per_block` set on packed ring groups. + """ + relevant = any( + (isinstance(spec, MambaSpec) and spec.tp_replicated) + or type(spec) is CircularBufferSpec + for worker_spec in kv_cache_specs + for spec in worker_spec.values() + ) + enabled = relevant and _side_cache_layout_enabled(vllm_config) + notes: list[str] = [] + if enabled: + global_kv_cache_groups, notes = _fold_replicated_mamba_groups( + global_kv_cache_groups, kv_cache_specs + ) + projected = [ + _project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec) + for worker_spec in kv_cache_specs + ] + if enabled and all( + _glm5_next_tensor_layout(groups) is None for groups in projected if groups + ): + for group_index, slots in _choose_ring_slots(projected).items(): + for groups in projected: + groups[group_index].slots_per_block = slots + notes.append( + f"QSA ring group {group_index} packs {slots} requests per pool block" + ) + if relevant and envs.VLLM_E2_SIDE_CACHE_LAYOUT and not enabled: + notes.append( + "not applicable to this engine (KV connector, HiSparse, DCP or a " + "layer-outermost layout)" + ) + if relevant: + # The A/B evidence line: boot logs must show which arm ran. + logger.info( + "Side-cache layout: %s (VLLM_E2_SIDE_CACHE_LAYOUT)%s", + "on" if enabled else "off", + "".join(f"; {note}" for note in notes), + ) + return global_kv_cache_groups, projected + + def get_kv_cache_configs( vllm_config: VllmConfig, kv_cache_specs: list[dict[str, KVCacheSpec]], @@ -2774,10 +3094,11 @@ def get_kv_cache_configs( # If original_max_model_len was -1, automatically # determine the maximum model length that fits in available GPU memory. # We use per-worker projected groups to account for PP sharding. - projected_groups_per_worker = [ - _project_kv_cache_groups_to_worker(global_kv_cache_groups, worker_spec) - for worker_spec in kv_cache_specs - ] + # v2.5: the side-cache layout (PLE fold, ring slots) is applied here, where + # every worker's layers are known, so no PP stage's block can widen. + global_kv_cache_groups, projected_groups_per_worker = _apply_side_cache_layout( + vllm_config, global_kv_cache_groups, kv_cache_specs + ) # If `num_gpu_blocks_override` is set, the cache size that will actually # be allocated is decoupled from the profiled `available_memory`: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index e5eb8bc1c1..4d5e490ce5 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -1619,6 +1619,10 @@ class Scheduler(SchedulerInterface): pending = set(block_ids) owners: dict[int, set[str]] = {} for mgr in self.kv_cache_manager.coordinator.single_type_managers: + if getattr(mgr, "slot_pool", None) is not None: + # v2.5 slot-packed ring: its block ids are slot ids, not pool + # block ids, and its pages are never zeroed. + continue for req_id, blocks in mgr.req_to_blocks.items(): for blk in blocks: if blk.block_id in pending: diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index ac7c9830f4..943fd39da1 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -1236,18 +1236,126 @@ class SlidingWindowManager(SingleTypeKVCacheManager): return 0 +class _RingSlotPage: + """One pool block shared by up to ``slots`` requests' rings.""" + + __slots__ = ("block", "free_slots", "live") + + def __init__(self, block: KVCacheBlock, slots: int) -> None: + self.block = block + # Highest first, so pop() hands out the lowest free slot. + self.free_slots = list(range(slots - 1, -1, -1)) + self.live = 0 + + +class RingSlotPool: + """v2.5 side-cache layout: ring blocks as slots of shared pool blocks. + + The QSA ring needs one block per request for the request's lifetime, but + its pages for every layer of a stage are ~16 KB against a 12.9 MB pool + block. This pool takes whole blocks from the main ``BlockPool`` and cuts + each into ``slots_per_block`` slots; a slot's block id is + ``pool_block_id * slots_per_block + slot``, which is exactly the index of + the slot in the worker's ring view (``KVCacheTensor.slots_per_block``). + + The slot blocks it hands out carry ``pool=self``, so every existing free + path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred + ``pop_blocks_for_free`` and the scheduler's fenced drain, which all call + ``BlockPool.free_blocks`` -- routes them back here, at the time the old + per-request block would have gone back (the deferred free is kept). A pool + block returns to the main pool when its last slot is freed. Rings are never + prefix-cached, so a slot block carries no hash. + """ + + def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None: + assert slots_per_block > 1 + self.block_pool = block_pool + self.slots_per_block = slots_per_block + # Pool block id -> page, in claim order (the oldest page fills first). + self._pages: dict[int, _RingSlotPage] = {} + + @property + def num_pages(self) -> int: + """Pool blocks currently held for rings.""" + return len(self._pages) + + @property + def num_live_slots(self) -> int: + return sum(page.live for page in self._pages.values()) + + def has_free_slot(self) -> bool: + return any(page.free_slots for page in self._pages.values()) + + def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]: + out: list[KVCacheBlock] = [] + for _ in range(num_blocks): + page = next((p for p in self._pages.values() if p.free_slots), None) + if page is None: + (pool_block,) = self.block_pool.get_new_blocks(1) + page = _RingSlotPage(pool_block, self.slots_per_block) + self._pages[pool_block.block_id] = page + slot = page.free_slots.pop() + page.live += 1 + out.append( + KVCacheBlock( + page.block.block_id * self.slots_per_block + slot, + ref_cnt=1, + pool=self, # type: ignore[arg-type] + ) + ) + return out + + def free_blocks(self, ordered_blocks) -> None: + released: list[KVCacheBlock] = [] + for block in ordered_blocks: + assert block.pool is self, "not a ring slot of this pool" + block.ref_cnt -= 1 + assert block.ref_cnt == 0, "ring slots have a single owner" + page_id, slot = divmod(block.block_id, self.slots_per_block) + page = self._pages[page_id] + assert slot not in page.free_slots, f"slot {block.block_id} freed twice" + page.free_slots.append(slot) + page.free_slots.sort(reverse=True) + page.live -= 1 + if page.live == 0: + del self._pages[page_id] + released.append(page.block) + if released: + self.block_pool.free_blocks(released) + + class CircularBufferManager(FullAttentionManager): """Claims the ring's single block per request; prefix caching disabled.""" supports_fine_grained_hash_lookup: ClassVar[bool] = False + # v2.5 side-cache layout: set by the coordinator for a slot-packed group. + _slot_pool: RingSlotPool | None = None + + def enable_slot_packing(self, slots_per_block: int) -> None: + """Claim ring slots of shared pool blocks instead of whole blocks.""" + assert not self.req_to_blocks, "enable slot packing before any claim" + self._slot_pool = RingSlotPool(self.block_pool, slots_per_block) + + @property + def slot_pool(self) -> RingSlotPool | None: + return self._slot_pool + + if self._slot_pool is not None: + new_blocks = self._slot_pool.get_new_blocks(1) + else: + new_blocks = self.block_pool.get_new_blocks(1) + def _claim_ring_block( self, request_id: str, record_for_zeroing: bool = True ) -> list[KVCacheBlock]: req_blocks = self.req_to_blocks[request_id] if req_blocks: return [] - new_blocks = self.block_pool.get_new_blocks(1) + if self._slot_pool is not None: + new_blocks = self._slot_pool.get_new_blocks(1) + else: + new_blocks = self.block_pool.get_new_blocks(1) req_blocks.extend(new_blocks) if self._record_new_block_ids and record_for_zeroing: self.new_block_ids.extend(block.block_id for block in new_blocks) @@ -1293,7 +1401,12 @@ class CircularBufferManager(FullAttentionManager): apply_admission_cap: bool = False, prefill_end: int = 0, ) -> int: - return 0 if self.req_to_blocks.get(request_id) else 1 + if self.req_to_blocks.get(request_id): + return 0 + # A free slot in a page the ring already holds costs no pool block. + if self._slot_pool is not None and self._slot_pool.has_free_slot(): + return 0 + return 1 def allocate_new_blocks( self, request_id: str, num_tokens: int, num_tokens_main_model: int diff --git a/vllm/v1/core/single_type_kv_cache_manager.py.orig b/vllm/v1/core/single_type_kv_cache_manager.py.orig new file mode 100644 index 0000000000..05a395375b --- /dev/null +++ b/vllm/v1/core/single_type_kv_cache_manager.py.orig @@ -0,0 +1,3004 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import itertools +from abc import ABC, abstractmethod +from collections import defaultdict +from collections.abc import Sequence +from typing import TYPE_CHECKING, ClassVar + +from vllm import envs +from vllm.distributed.kv_events import MEDIUM_CPU +from vllm.logger import init_logger +from vllm.utils.math_utils import cdiv +from vllm.v1.core.block_pool import BlockPool +from vllm.v1.core.kv_cache_utils import ( + BlockHashList, + BlockHashListWithBlockSize, + BlockHashWithGroupId, + KVCacheBlock, + resolve_block_hashes, +) +from vllm.v1.hisparse.block_pool import SharedEventQueueBlockPool +from vllm.v1.kv_cache_interface import ( + AttentionSpec, + ChunkedLocalAttentionSpec, + CircularBufferSpec, + CrossAttentionSpec, + FullAttentionSpec, + HiddenStateCacheSpec, + HiSparseHotSpec, + HiSparseResidentSpec, + KpoolTailSpec, + KVCacheGroupRole, + KVCacheSpec, + MambaSpec, + MLAAttentionSpec, + RSWASpec, + SinkFullAttentionSpec, + SlidingWindowMLASpec, + SlidingWindowSpec, + get_mamba_prefill_checkpoint_position, + is_mamba_prefill_checkpoint_valid, +) +from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry +from vllm.v1.request import Request + +if TYPE_CHECKING: + from vllm.v1.hisparse.coordinator import HiSparseCoordinator + +logger = init_logger(__name__) + + +class SingleTypeKVCacheManager(ABC): + """An abstract base class for a manager that handle the kv cache management + logic of one specific type of attention layer. + """ + + supports_fine_grained_hash_lookup: ClassVar[bool] = False + + # Keep this group's longer prefix until external cache lookup completes. + retains_longer_hit: bool = False + + @property + def has_positionally_stable_blocks(self) -> bool: + """Whether positional offload scans can follow this block table. + + True means already-scanned indices will not be reused for a different + token range. Managers may still null blocks that left their retention + window, as long as the cursor's positional history remains valid. + """ + return True + + def __init__( + self, + kv_cache_spec: KVCacheSpec, + block_pool: BlockPool, + enable_caching: bool, + kv_cache_group_id: int, + scheduler_block_size: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + needs_kv_cache_zeroing: bool = False, + max_admission_blocks_per_request: int | None = None, + ) -> None: + """Initializes the SingleTypeKVCacheManager. + + Args: + kv_cache_spec: The kv_cache_spec for this manager. + block_pool: The block pool. + enable_caching: Whether prefix caching is enabled. + kv_cache_group_id: The id of the kv cache group of this manager. + scheduler_block_size: The scheduling granularity (LCM of all group + block sizes); a multiple of this manager's ``block_size``. + dcp_world_size: Decode context parallel world size. + pcp_world_size: Prefill context parallel world size. + needs_kv_cache_zeroing: Whether worker-side KV cache zeroing needs + newly allocated block IDs from this manager. + max_admission_blocks_per_request: Recycling-aware per-request + block cap used by `get_num_blocks_to_allocate`. Only set for + spec types that recycle blocks across chunks (SWA, + chunked-local); `None` (the default) means no cap, which is + correct for full-attention-style specs that hold every + block until the request finishes. + + """ + self.scheduler_block_size = scheduler_block_size + # Hybrid fine-grained lookup may lower this after all participating + # managers have been validated by the coordinator. + self.cache_hit_alignment_tokens = scheduler_block_size + # The block size for this manager; used for actual block allocation. + self.dcp_world_size = dcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.pcp_world_size = pcp_world_size if kv_cache_spec.dcp_sharded else 1 + self.block_size = kv_cache_spec.block_size * self.dcp_world_size + self.kv_cache_spec = kv_cache_spec + self.block_pool = block_pool + self.enable_caching = enable_caching + self.max_admission_blocks_per_request = max_admission_blocks_per_request + # Record newly allocated block ids only when worker-side zeroing will + # consume them and this manager holds a spec type that gets zeroed. + self._record_new_block_ids = ( + needs_kv_cache_zeroing + and isinstance(kv_cache_spec, AttentionSpec) + and not isinstance(kv_cache_spec, CircularBufferSpec) + ) + self.new_block_ids: list[int] = [] + # v2.2 ring-row tag fix: ring blocks claimed since the last drain. The ring stays + # OUT of whole-block zeroing above; instead every claim is recorded + # UNCONDITIONALLY (never under needs_kv_cache_zeroing) so the worker + # clears the rows' tag words before the new owner's first step. Only + # CircularBufferManager appends here. + self.new_ring_block_ids: list[int] = [] + + # Mapping from request ID to blocks to track the blocks allocated + # for each request, so that we can free the blocks when the request + # is finished. + self.req_to_blocks: defaultdict[str, list[KVCacheBlock]] = defaultdict(list) + + # {req_id: The number of cached blocks for this given request} + # This is used to track the number of cached blocks for each request. + # This is only used to track the RUNNING requests, we do not track the + # data for preempted ones. + self.num_cached_block: dict[str, int] = {} + + self.kv_cache_group_id = kv_cache_group_id + self._null_block = block_pool.null_block + + # Whether this group's prefix-cache hits drop the EAGLE/MTP lookahead + # block. Only consulted by managers whose hit logic is sparse within an + # aligned segment (SWA). Initialized lazily by the coordinator after + # determining the attention groups. + self.use_eagle = False + # ``CacheConfig.enable_mamba_shared_prefix_checkpoint``, narrowed and set + # by ``KVCacheManager``; only an EAGLE Mamba "align" group ever gets it. + self.shared_prefix_checkpoint = False + # Partial-hit copy-on-write bookkeeping. Populated only by fine-grained + # managers (full attention, mamba "align"); harmlessly empty elsewhere. + self._partial_hit_reqs: dict[str, tuple[int, KVCacheBlock]] = {} + self._pending_cow_copies: list[tuple[KVCacheBlock, KVCacheBlock]] = [] + # Boundary-state offload hand-off for external KV connectors. A mamba + # "align" block table is not append-only (interior states are + # nulled/freed and speculative blocks relocate in place), so a + # connector cannot resolve its state blocks positionally. Record + # (request, group, block, exact token boundary) for each committed + # boundary state so a connector can offload the right block under the + # right hash. Populated only by mamba "align". + self._pending_boundary_state_offloads: list[ + tuple[str, int, KVCacheBlock, int] + ] = [] + + @classmethod + def _get_num_evictable_blocks(cls, blocks: Sequence[KVCacheBlock]): + return sum(blk.ref_cnt == 0 and not blk.is_null for blk in blocks) + + def _has_partial_local_hit( + self, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + ) -> bool: + # The local prefix-cache hit ends inside one of this manager's + # blocks: the shared tail block needs CoW. + return ( + len(new_computed_blocks) > 0 + and num_local_computed_tokens % self.block_size != 0 + ) + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + prefill_end: int = 0, + ) -> int: + """Get the number of blocks needed to be allocated for the request. + + Args: + request_id: The request ID. + num_tokens: The total number of tokens that need a slot (including + tokens that are already allocated). + new_computed_blocks: The new computed blocks just hitting the + prefix caching. + total_computed_tokens: Include both local and external computed + tokens. + num_local_computed_tokens: The number of local prefix-cache computed + tokens. + num_tokens_main_model: The number of tokens for the main model (aka target + model in spec decode). w/o spec decode, it is num_tokens; + with spec decode, it is num_tokens - num_lookahead_tokens. + apply_admission_cap: If True, clamp by `num_required_blocks` by + `max_admission_blocks_per_request`for recycling-aware specs + (SWA, chunked-local). + prefill_end: The token index the request's prefill ends at, the + same value the scheduler splits chunks against. Mamba reserves + a prefill checkpoint only on the chunk reaching it; 0 (dense + retention) reserves one on every chunk. + + Returns: + The number of blocks to allocate. + + """ + num_required_blocks = cdiv(num_tokens, self.block_size) + if apply_admission_cap and self.max_admission_blocks_per_request is not None: + # Recycling-aware specs (SWA, chunked-local) cap the per-request + # reservation here so admission matches the startup pool sizer + # (`SlidingWindowSpec.max_admission_blocks_per_request` / its + # chunked-local counterpart). `remove_skipped_blocks` runs from + # `allocate_slots` before each chunk's `get_num_blocks_to_allocate`, + # so per-request peak real-held blocks <= this cap, which keeps + # `sum(reservations) <= pool` <=> `sum(peak_real_held) <= pool`. + # Drift between the two would re-introduce the deadlock from + # issue #39734 or, worse, mid-prefill OOM. + num_required_blocks = min( + num_required_blocks, self.max_admission_blocks_per_request + ) + num_req_blocks = len(self.req_to_blocks.get(request_id, ())) + + if request_id in self.num_cached_block: + # Fast-path: a running request won't have any new prefix-cache hits. + assert len(new_computed_blocks) == 0 + # NOTE: With speculative decoding, request's blocks may be allocated + # for draft tokens which are later rejected. In this case, + # num_required_blocks may be smaller than num_req_blocks. + return max(num_required_blocks - num_req_blocks, 0) + + num_skipped_tokens = self.get_num_skipped_tokens(total_computed_tokens) + num_local_computed_blocks = len(new_computed_blocks) + num_req_blocks + # Number of whole blocks that are skipped by the attention window. + # If nothing is skipped, this is 0. + num_skipped_blocks = num_skipped_tokens // self.block_size + # We need blocks for the non-skipped suffix. If there are still + # local-computed blocks inside the window, they contribute to the + # required capacity; otherwise, skipped blocks dominate. + num_new_blocks = max( + num_required_blocks - max(num_skipped_blocks, num_local_computed_blocks), + 0, + ) + + # Among the `new_computed_blocks`, the first `num_skipped_blocks` worth + # of blocks are skipped; `num_req_blocks` of those may already be in + # `req_to_blocks`, so only skip the remainder from `new_computed_blocks`. + num_skipped_new_computed_blocks = max(0, num_skipped_blocks - num_req_blocks) + + # If a computed block is an eviction candidate (in the free queue and + # ref_cnt == 0), it will be removed from the free queue when touched by + # the allocated request, so we must count it in the free-capacity check. + num_evictable_blocks = self._get_num_evictable_blocks( + new_computed_blocks[num_skipped_new_computed_blocks:] + ) + if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens): + # Reserve the extra block that allocate_new_blocks pulls for the + # partial-hit CoW redirect. + num_new_blocks += 1 + return num_new_blocks + num_evictable_blocks + + def add_local_computed_blocks( + self, + request_id: str, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + """Add the locally cached (prefix-hit) blocks to the request: + 1. Touch the computed blocks (paired with adding them to `req_blocks`) + so their ref_cnt exactly tracks the referencing requests. + 1.5. (Optional) For sliding window, skipped blocks are padded with nulls. + 2. Add the remaining computed blocks. + + Args: + request_id: The request ID. + new_computed_blocks: The new computed blocks just hitting the + prefix cache. + num_local_computed_tokens: The number of local computed tokens. + num_external_computed_tokens: The number of external computed tokens. + + """ + # The coordinator only calls this for first-time allocations (running + # requests are short-circuited there), so the request has no blocks yet. + req_blocks = self.req_to_blocks[request_id] + assert len(req_blocks) == 0 + num_total_computed_tokens = ( + num_local_computed_tokens + num_external_computed_tokens + ) + num_skipped_tokens = self.get_num_skipped_tokens(num_total_computed_tokens) + num_skipped_blocks = num_skipped_tokens // self.block_size + if num_skipped_blocks > 0: + # It is possible that all new computed blocks are skipped when + # num_skipped_blocks > len(new_computed_blocks). + new_computed_blocks = new_computed_blocks[num_skipped_blocks:] + + # Touch the computed blocks to make sure they won't be evicted. + if self.enable_caching: + self.block_pool.touch(new_computed_blocks) + else: + assert not any(new_computed_blocks), ( + "Computed blocks should be empty when prefix caching is disabled" + ) + + # Skip blocks are padded with null blocks. + req_blocks.extend([self._null_block] * num_skipped_blocks) + # Add the remaining computed blocks. + req_blocks.extend(new_computed_blocks) + # All cached hits (including skipped nulls) are already cached; mark + # them so cache_blocks() will not try to re-cache blocks that already + # have a block_hash set. + self.num_cached_block[request_id] = len(req_blocks) + if self._has_partial_local_hit(new_computed_blocks, num_local_computed_tokens): + # Record the partial tail for the CoW redirect in + # allocate_new_blocks; cap the cached count at the full blocks so + # cache_blocks() re-caches the private copy once full. + block_idx = num_local_computed_tokens // self.block_size + self._partial_hit_reqs[request_id] = (block_idx, new_computed_blocks[-1]) + self.num_cached_block[request_id] = block_idx + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + record_for_zeroing: bool = True, + ) -> None: + """Allocate new blocks for external (KV-connector) computed tokens. + + Must run only after every group's local blocks have been touched via + `add_local_computed_blocks`, so this group's `get_new_blocks` cannot + evict another group's cache-hit blocks (issue #33775). + + Args: + request_id: The request ID. + num_local_computed_tokens: The number of local computed tokens. + num_external_computed_tokens: The number of external computed tokens. + record_for_zeroing: Whether the new blocks need zeroing. False when + the load writes them after this step, which zeroing would race. + + """ + num_total_computed_tokens = ( + num_local_computed_tokens + num_external_computed_tokens + ) + num_skipped_tokens = self.get_num_skipped_tokens(num_total_computed_tokens) + if num_skipped_tokens > 0: + # Some external computed tokens may be skipped too. + num_external_computed_tokens = min( + num_total_computed_tokens - num_skipped_tokens, + num_external_computed_tokens, + ) + if num_external_computed_tokens <= 0: + return + + req_blocks = self.req_to_blocks[request_id] + num_new_blocks = max( + 0, cdiv(num_total_computed_tokens, self.block_size) - len(req_blocks) + ) + allocated_blocks = self.block_pool.get_new_blocks(num_new_blocks) + req_blocks.extend(allocated_blocks) + if self._record_new_block_ids and record_for_zeroing: + self.new_block_ids.extend(b.block_id for b in allocated_blocks) + + def allocate_new_blocks( + self, request_id: str, num_tokens: int, num_tokens_main_model: int + ) -> list[KVCacheBlock]: + """Allocate new blocks for the request to give it at least `num_tokens` + token slots. + + Args: + request_id: The request ID. + num_tokens: The total number of tokens that need a slot (including + tokens that are already allocated). + num_tokens_main_model: The number of tokens for the main model (aka target + model in spec decode). w/o spec decode, it is num_tokens; + with spec decode, it is num_tokens - num_lookahead_tokens. + + Returns: + The new allocated blocks. + + """ + cow_blocks: list[KVCacheBlock] = [] + if request_id in self._partial_hit_reqs: + # Partial hit: redirect the shared tail to a private CoW block. + # Replacing in place keeps the length-based allocation below + # correct; the extra block was reserved by + # get_num_blocks_to_allocate. + block_idx, source_block = self._partial_hit_reqs.pop(request_id) + cow_block = self.block_pool.get_new_blocks(1)[0] + self._apply_cow(request_id, block_idx, source_block, cow_block) + self.new_block_ids.append(cow_block.block_id) + cow_blocks.append(cow_block) + + req_blocks = self.req_to_blocks[request_id] + num_required_blocks = cdiv(num_tokens, self.block_size) + num_new_blocks = num_required_blocks - len(req_blocks) + if num_new_blocks <= 0: + return cow_blocks + else: + new_blocks = self.block_pool.get_new_blocks(num_new_blocks) + req_blocks.extend(new_blocks) + if self._record_new_block_ids: + self.new_block_ids.extend(b.block_id for b in new_blocks) + return cow_blocks + new_blocks + + @property + def records_new_block_ids(self) -> bool: + """Whether this manager's new blocks are zeroed by the worker.""" + return self._record_new_block_ids + + def take_new_block_ids(self) -> list[int]: + """Drain and return block IDs allocated since the last call.""" + ids = self.new_block_ids + self.new_block_ids = [] + return ids + + def take_new_ring_block_ids(self) -> list[int]: + """Drain ring blocks claimed since the last call; empty for + every manager but CircularBufferManager.""" + ids = self.new_ring_block_ids + self.new_ring_block_ids = [] + return ids + + def get_ring_block_ids(self, request_id: str) -> list[int]: + """The request's ring block ids; empty for non-ring managers.""" + return [] + + def take_pending_cow_copies( + self, + ) -> list[tuple[KVCacheBlock, KVCacheBlock]]: + """Drain pending CoW source and destination block pairs.""" + pending_copies = self._pending_cow_copies + self._pending_cow_copies = [] + return pending_copies + + def take_pending_boundary_state_offloads( + self, + ) -> list[tuple[str, int, KVCacheBlock, int]]: + """Drain producer boundary-state hand-offs. + + Entries are ``(req_id, group_id, block, boundary_tokens)``. + + Only mamba "align" populates this. The blocks are not kept alive by + the request block table for the whole request, so a caller that reads + them asynchronously must pin them first. + """ + pending = self._pending_boundary_state_offloads + self._pending_boundary_state_offloads = [] + return pending + + def finalize_partial_tail_offload( + self, + request_id: str, + num_computed_tokens: int, + num_in_flight_tokens: int, + ) -> tuple[int, KVCacheBlock, int] | None: + """Finalize a producer partial tail when its request finishes.""" + return None + + def _apply_cow( + self, + request_id: str, + block_idx: int, + source_block: KVCacheBlock, + cow_block: KVCacheBlock, + ) -> None: + """Redirect a partial prefix-cache hit to a private CoW block. + + Both copy endpoints stay retained until the copy has run on the worker, + so a same-step free cannot recycle them: ``source_block`` keeps its + hit-ref, ``cow_block`` takes an extra ref beyond the one handed to the + request. + """ + req_blocks = self.req_to_blocks[request_id] + assert block_idx < len(req_blocks) + assert req_blocks[block_idx] is source_block + assert not source_block.is_null and source_block.ref_cnt > 0 + req_blocks[block_idx] = cow_block + self._pending_cow_copies.append((source_block, cow_block)) + cow_block.ref_cnt += 1 + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + """Cache the blocks for the request. + + Args: + request: The request. + num_tokens: The total number of tokens that need to be cached + (including tokens that are already cached). + retention_interval: Sparse local-checkpoint granularity. ``None`` + keeps dense checkpointing; ``0`` keeps only the latest replay + boundary; a positive multiple of ``scheduler_block_size`` keeps + a tail once per that-sized segment. Only SWA acts on it. + replay_boundaries: Positions a later request replaying this prompt + can resume at, from ``get_replay_boundaries``. + + """ + if not self.kv_cache_spec.prefix_cacheable: + return + num_cached_blocks = self.num_cached_block.get(request.request_id, 0) + num_full_blocks = num_tokens // self.block_size + + if num_cached_blocks >= num_full_blocks: + return + + # Token boundaries whose reachable tail must be retained under sparse + # retention: every position a replaying sibling can resume at (see + # ``get_replay_boundaries``) and any detected shared-prefix junction. + reachable_boundaries = [*replay_boundaries] + if request.shared_prefix_boundary: + reachable_boundaries.append(request.shared_prefix_boundary) + + block_mask = self.reachable_block_mask( + start_block=num_cached_blocks, + end_block=num_full_blocks, + alignment_tokens=self.cache_hit_alignment_tokens, + kv_cache_spec=self.kv_cache_spec, + use_eagle=self.use_eagle, + retention_interval=retention_interval, + reachable_boundaries=reachable_boundaries, + dcp_world_size=self.dcp_world_size, + ) + self.block_pool.cache_full_blocks( + request=request, + blocks=self.req_to_blocks[request.request_id], + num_cached_blocks=num_cached_blocks, + num_full_blocks=num_full_blocks, + block_size=self.block_size, + kv_cache_group_id=self.kv_cache_group_id, + block_mask=block_mask, + ) + + self.num_cached_block[request.request_id] = num_full_blocks + + @classmethod + def reachable_block_mask( + cls, + start_block: int, + end_block: int, + alignment_tokens: int | None, + kv_cache_spec: KVCacheSpec, + use_eagle: bool, + retention_interval: int | None = None, + reachable_boundaries: Sequence[int] = (), + dcp_world_size: int = 1, + final_segment_end_block: int | None = None, + ) -> list[bool] | None: + """Per-block mask for ``cache_full_blocks``. ``None`` means cache + every (non-null) block — the default for full attention. + + Subclasses with sparse hit semantics (SWA / Mamba) override this to skip + blocks that can never serve a hit at any alignment-aligned prefix length. + ``reachable_boundaries`` are token positions whose reachable tail must be + retained; the base (dense) policy ignores them. + ``final_segment_end_block`` is the exclusive end of the request's final + segment. It may be later than ``end_block`` while prefill is still in + progress. Non-EAGLE sparse managers may keep its reachable tail. The + base policy already keeps every block, so it ignores this value. + """ + return None + + def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: + """Pop the request's bookkeeping and return its blocks without yet + returning them to the block pool. The caller is responsible for + eventually passing the returned blocks to `block_pool.free_blocks`, + freeing them in reverse order (so that tail blocks are evicted first). + + Args: + request_id: The request ID. + + Returns: + The request's blocks in allocation order. + + """ + # Default to [] in case a request is freed (aborted) before alloc. + req_blocks = self.req_to_blocks.pop(request_id, []) + self.num_cached_block.pop(request_id, None) + self._partial_hit_reqs.pop(request_id, None) + return req_blocks + + def free(self, request_id: str) -> None: + """Free the blocks for the request. + + Args: + request_id: The request ID. + + """ + # Free blocks in reverse order so that the tail blocks are freed first. + self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id))) + + def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]: + """Pop the prefix-cache checkpoints held for ``request_id`` outside its + block table, without freeing them. Only align-mode Mamba holds any + (see ``MambaManager._retire_state_blocks``). The coordinator frees them + after every group's table, so they are the request's last blocks to + become eviction candidates.""" + return [] + + def get_num_retained_checkpoint_blocks_to_reserve( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock] = (), + ) -> int: + """Blocks the admission gate must add for checkpoints this request + will hold outside its block table (0 for every manager but align-mode + Mamba with retention on).""" + return 0 + + @abstractmethod + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + """Get the number of common prefix blocks for all requests with allocated + KV cache. + + Args: + running_request_id: The request ID. + + Returns: + The number of common prefix blocks for all requests with allocated + KV cache. + + """ + raise NotImplementedError + + @classmethod + @abstractmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + """Get the longest cache hit prefix of the blocks that is not longer than + `max_length`. The prefix should be a common prefix hit for all the + kv cache groups in `kv_cache_group_ids`. If no cache hit is found, + return an empty list. + If eagle is enabled, drop the last matched block to force recompute the + last block to get the required hidden states for eagle drafting head. + For multi-module MTP, this recompute also rewrites the dropped block's + draft-layer KVs, which depend on up to num_speculative_tokens - 1 + tokens past the matched prefix (i.e. on the cache writer's + continuation, which the block hash does not cover); the coordinator + asserts the block size covers that window. + Need to be customized for each attention type. + + Args: + block_hashes: The block hashes of the request. + max_length: The maximum length of the cache hit prefix. + kv_cache_group_ids: The ids of the kv cache groups. + block_pool: The block pool. + kv_cache_spec: The kv cache spec. + drop_eagle_block: Whether to drop the last matched block for EAGLE/MTP. + Always False for non-EAGLE/MTP groups, but can be False for EAGLE/MTP + groups too if the last block is already dropped (e.g., in a + convergence loop in `find_longest_cache_hit`). + alignment_tokens: The returned cache hit length (in tokens) should + be a multiple of this value (in tokens). By default, it should + be set to the block_size. + dcp_world_size: The world size of decode context parallelism. + pcp_world_size: The world size of prefill context parallelism. + + Returns: + A tuple containing cached blocks and the exact cache-hit length in + tokens. The cached block tuple has skipped blocks replaced by null + blocks for each kv cache group in `kv_cache_group_ids`. + For example, sliding window manager should return a list like + ([NULL, NULL, KVCacheBlock(7), KVCacheBlock(8)]) for block size 4 + and sliding window 8 and len(kv_cache_group_ids) = 1. + + """ + raise NotImplementedError + + def _remove_blocks_in_range( + self, + request_id: str, + first_block: int, + last_block: int, + ) -> None: + """Free blocks in ``[first_block, last_block)`` and replace with null_block. + + Iterates backward so newly-evictable tail blocks are reached even after + earlier blocks in the range were nulled in a prior call. + """ + if request_id not in self.req_to_blocks: + return + if first_block >= last_block: + return + blocks = self.req_to_blocks[request_id] + last_block = min(last_block, len(blocks)) + + freed: list[KVCacheBlock] = [] + for i in range(last_block - 1, first_block - 1, -1): + if blocks[i] == self._null_block: + break + freed.append(blocks[i]) + blocks[i] = self._null_block + if freed: + self.block_pool.free_blocks(freed) + + def remove_skipped_blocks( + self, + request_id: str, + processed_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + """Remove and free blocks no longer needed for attention computation. + + The removed blocks should be replaced by null_block. + + This function depends on `get_num_skipped_tokens`, which need to be implemented + differently for each attention type. + + Args: + request_id: The request ID. + processed_computed_tokens: Computed-token prefix length covering + fully processed and committed tokens only (safe to free). + num_prompt_tokens: Optional prompt length for attention types (e.g. + R-SWA) that evict a middle gap rather than a head prefix. Ignored + by the default implementation. + + """ + del num_prompt_tokens + # Remove the blocks that will be skipped during attention computation. + num_skipped_tokens = self.get_num_skipped_tokens(processed_computed_tokens) + if num_skipped_tokens <= 0: + # This indicates that ALL tokens are inside attention window. + # Thus we do not need to free any blocks outside attention window. + # A typical case is full attention that we never free any token + # before the request is finished. + return + blocks = self.req_to_blocks[request_id] + num_skipped_blocks = num_skipped_tokens // self.block_size + # `num_skipped_tokens` may include tokens that haven't been allocated yet + # (e.g., when the attention window moves into the external computed tokens + # range), so we must cap to the number of blocks that currently exist for + # this request. + num_skipped_blocks = min(num_skipped_blocks, len(blocks)) + self._remove_blocks_in_range(request_id, 0, num_skipped_blocks) + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + """Get the number of tokens that will be skipped for attention computation. + + Args: + num_computed_tokens: The number of tokens that have been computed. + + Returns: + The number of tokens that will be skipped for attention computation. + + """ + # The default behavior is to not skip any tokens. + return 0 + + def new_step_starts(self) -> None: + return None + + +class FullAttentionManager(SingleTypeKVCacheManager): + supports_fine_grained_hash_lookup: ClassVar[bool] = True + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + assert isinstance( + kv_cache_spec, FullAttentionSpec | ChunkedLocalAttentionSpec + ), ( + "FullAttentionManager can only be used for full attention " + "and chunked local attention groups" + ) + block_size = kv_cache_spec.block_size + if dcp_world_size > 1: + # DCP shards each block's KV across ranks; hashes must be viewed at + # the sharded block size. + block_size *= dcp_world_size + block_hashes = resolve_block_hashes( + block_hashes, + block_pool.hash_block_size, + block_size, + supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup, + alignment_tokens=alignment_tokens, + ) + + # Fine-grained mode (alignment_tokens == hash_block_size < + # block_size): resolve_block_hashes kept the raw hash-granularity + # list so interior boundaries can be probed. + fine_grained = ( + alignment_tokens < block_size and block_size % alignment_tokens == 0 + ) + if fine_grained: + # list or lazy BlobBlockHashes view + assert isinstance(block_hashes, Sequence) + full_block_hashes: BlockHashList = BlockHashListWithBlockSize( + block_hashes, alignment_tokens, block_size + ) + else: + full_block_hashes = block_hashes + + computed_blocks: tuple[list[KVCacheBlock], ...] = tuple( + [] for _ in range(len(kv_cache_group_ids)) + ) + # Phase 1: longest run of cached full blocks from the start. A missing + # block implies every later block misses too (chained hashes). + for block_hash in itertools.islice(full_block_hashes, max_length // block_size): + cached_block = block_pool.get_cached_block(block_hash, kv_cache_group_ids) + if not cached_block: + break + for computed, cached in zip(computed_blocks, cached_block): + computed.append(cached) + hit_length = len(computed_blocks[0]) * block_size + + # Phase 2 (fine-grained only): extend into the first non-full block by + # probing its interior hash boundaries high-to-low (longest hit first). + if fine_grained: + # list or lazy BlobBlockHashes view + assert isinstance(block_hashes, Sequence) + scale_factor = block_size // alignment_tokens + first_partial_idx = len(computed_blocks[0]) * scale_factor + max_partial_idx = min( + first_partial_idx + scale_factor - 1, + max_length // alignment_tokens, + len(block_hashes), + ) + for fine_idx in range(max_partial_idx - 1, first_partial_idx - 1, -1): + cached_tail = block_pool.get_cached_block( + block_hashes[fine_idx], kv_cache_group_ids + ) + if not cached_tail: + continue + for computed, cached in zip(computed_blocks, cached_tail): + computed.append(cached) + hit_length = (fine_idx + 1) * alignment_tokens + break + + # Eagle needs the tokens right before the generation point recomputed: + # drop one hash unit when fine-grained (the tail block's KV is + # append-only, so it still covers the reduced length), else one cache + # block. + if drop_eagle_block and hit_length > 0: + hit_length -= min(alignment_tokens, block_size) + # Round down to the alignment; a no-op when fine-grained (hits land on + # hash boundaries by construction) and when alignment_tokens == + # block_size. Then trim blocks past the new tail. + hit_length -= hit_length % alignment_tokens + num_blocks = cdiv(hit_length, block_size) + for computed in computed_blocks: + del computed[num_blocks:] + return computed_blocks, hit_length + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + super().cache_blocks( + request, + num_tokens, + retention_interval=retention_interval, + replay_boundaries=replay_boundaries, + ) + hash_block_size = self.block_pool.hash_block_size + if self.block_size == hash_block_size: + return + self._cache_partial_tail_block(request, num_tokens) + + def _cache_partial_tail_block( + self, + request: Request, + num_tokens: int, + ) -> None: + """Cache the prompt tail when it ends inside a cache block. + + Only the final prompt hash boundary is registered as a partial + prefix-cache entry; intermediate hash boundaries inside the same cache + block are intentionally skipped. + """ + hash_block_size = self.block_pool.hash_block_size + boundary_tokens = request.num_prompt_tokens // hash_block_size * hash_block_size + if boundary_tokens == 0 or boundary_tokens > num_tokens: + return + if boundary_tokens % self.block_size == 0: + return + + blocks = self.req_to_blocks[request.request_id] + block_idx = boundary_tokens // self.block_size + if block_idx >= len(blocks): + return + self.block_pool.cache_partial_block( + request=request, + block=blocks[block_idx], + num_tokens=boundary_tokens, + kv_cache_group_id=self.kv_cache_group_id, + block_size=self.block_size, + ) + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + blocks = self.req_to_blocks[running_request_id] + num_common_blocks = 0 + for block in blocks: + if block.ref_cnt == len(self.req_to_blocks): + num_common_blocks += 1 + else: + break + return num_common_blocks + + +class RSWAManager(FullAttentionManager): + """KV cache manager for Reference Sliding Window Attention (R-SWA). + + When ``num_prompt_tokens`` is supplied to ``remove_skipped_blocks``, frees + gap blocks between the prefill tail and the current decode window. This + bounds per-request KV memory at O(prefix_len + rswa_window) instead of + growing linearly with decode length. + """ + + def __init__(self, kv_cache_spec: RSWASpec, **kwargs) -> None: + super().__init__(kv_cache_spec, **kwargs) + self.rswa_window: int = kv_cache_spec.rswa_window + + def remove_skipped_blocks( + self, + request_id: str, + processed_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + """Free gap blocks that are no longer needed for attention. + + Gap = blocks entirely within + [ceil(prefix_len / block_size) * block_size, + max(prefix_len, processed_computed_tokens - rswa_window)) + + Freed blocks are replaced with null_block in req_to_blocks so the + block_table passed to FA4 is valid (null_block KV is all-zero; + rswa_mask_mod marks gap positions as non-visible so FA4 skips them). + """ + if num_prompt_tokens is None: + super().remove_skipped_blocks( + request_id, processed_computed_tokens, num_prompt_tokens + ) + return + + bs = self.block_size + # First block fully after the prefill boundary. + first_gap_block = cdiv(num_prompt_tokens, bs) + # Decode window start position; blocks before this are evictable. + window_start = max( + num_prompt_tokens, processed_computed_tokens - self.rswa_window + ) + last_gap_block = window_start // bs # exclusive upper bound + self._remove_blocks_in_range(request_id, first_gap_block, last_gap_block) + + +class SlidingWindowManager(SingleTypeKVCacheManager): + def __init__(self, kv_cache_spec: SlidingWindowSpec, **kwargs) -> None: + super().__init__(kv_cache_spec, **kwargs) + self.sliding_window = kv_cache_spec.sliding_window + # Extra trailing tokens to retain below the window (never attended) so a + # multi-module MTP store-side lag can still reconstruct the window from + # cached blocks. + self.extra_retained_tokens = kv_cache_spec.extra_retained_tokens + + @classmethod + def _contiguous_blocks_for_hit( + cls, window_size: int, block_size: int, use_eagle: bool + ) -> int: + blocks = cdiv(window_size - 1, block_size) + if use_eagle: + # Need to drop the last matched block if eagle is enabled. For + # sliding window layer, we achieve this by increasing the number of + # contiguous blocks needed for prefix cache hit by one and dropping + # the last matched block. + blocks += 1 + return blocks + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + assert isinstance(kv_cache_spec, SlidingWindowSpec), ( + "SlidingWindowManager can only be used for sliding window groups" + ) + assert dcp_world_size == 1, "DCP not support sliding window attn now." + assert pcp_world_size == 1, "PCP not support sliding window attn now." + # Sliding-window cache hits must stay at the group's physical block + # granularity. resolve_block_hashes() converts finer-grained hashes to + # that view when the hybrid-cache alignment is smaller than block_size. + block_hashes = resolve_block_hashes( + block_hashes, + block_pool.hash_block_size, + kv_cache_spec.block_size, + supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup, + alignment_tokens=alignment_tokens, + ) + + # The number of contiguous blocks needed for a prefix cache hit. + sliding_window_contiguous_blocks = cls._contiguous_blocks_for_hit( + kv_cache_spec.sliding_window, kv_cache_spec.block_size, drop_eagle_block + ) + + # TODO: reduce i by sliding_window_contiguous_blocks when cache miss, to + # optimize the time complexity from O(max_num_blocks) to + # O(max_num_blocks / sliding_window_contiguous_blocks + + # sliding_window_contiguous_blocks), + # which is good for low cache hit rate scenarios. + max_num_blocks = max_length // kv_cache_spec.block_size + computed_blocks: tuple[list[KVCacheBlock], ...] = tuple( + [block_pool.null_block] * max_num_blocks + for _ in range(len(kv_cache_group_ids)) + ) + block_size = kv_cache_spec.block_size + num_contiguous_blocks = 0 + match_found = False + # Search from right to left and early stop when a match is found. + for i in range(max_num_blocks - 1, -1, -1): + if cached_block := block_pool.get_cached_block( + block_hashes[i], kv_cache_group_ids + ): + # Skip prefix matching check if the block is not aligned with + # `alignment_tokens`. + if num_contiguous_blocks == 0 and block_size != alignment_tokens: + post_pop_blocks = i if drop_eagle_block else i + 1 + if (post_pop_blocks * block_size) % alignment_tokens != 0: + continue + # Add the cached block to the computed blocks. + for computed, cached in zip(computed_blocks, cached_block): + computed[i] = cached + num_contiguous_blocks += 1 + if num_contiguous_blocks >= sliding_window_contiguous_blocks: + # Trim the trailing blocks. + # E.g., [NULL, NULL, 8, 3, NULL, 9] -> [NULL, NULL, 8, 3] + # when sliding_window_contiguous_blocks=2. + for computed in computed_blocks: + del computed[i + num_contiguous_blocks :] + match_found = True + break + else: + num_contiguous_blocks = 0 + if not match_found: + # The first `num_contiguous_blocks` is a cache hit even if + # `num_contiguous_blocks < sliding_window_contiguous_blocks`. + for computed in computed_blocks: + del computed[num_contiguous_blocks:] + while ( + block_size != alignment_tokens # Faster for common case. + and len(computed_blocks[0]) * block_size % alignment_tokens != 0 + ): + for computed in computed_blocks: + computed.pop() + if drop_eagle_block and computed_blocks[0]: + for computed in computed_blocks: + computed.pop() + # Re-align after eagle pop: the pop may break the alignment + # when block_size != alignment_tokens (hybrid models with + # different page sizes, e.g. Gemma4). + while ( + block_size != alignment_tokens + and len(computed_blocks[0]) * block_size % alignment_tokens != 0 + ): + for computed in computed_blocks: + computed.pop() + hit_length = len(computed_blocks[0]) * block_size + return computed_blocks, hit_length + + @classmethod + def reachable_block_mask( + cls, + start_block: int, + end_block: int, + alignment_tokens: int | None, + kv_cache_spec: KVCacheSpec, + use_eagle: bool, + retention_interval: int | None = None, + reachable_boundaries: Sequence[int] = (), + dcp_world_size: int = 1, + final_segment_end_block: int | None = None, + ) -> list[bool] | None: + assert isinstance(kv_cache_spec, SlidingWindowSpec) + if alignment_tokens is None: + # Fast path: when the coordinator imposes no alignment constraint. + return None + block_size = kv_cache_spec.block_size * dcp_world_size + if alignment_tokens % block_size != 0: + # The mask is block-granular, so a sub-block alignment cannot be + # represented exactly. This happens for hybrid offloading, where + # ``alignment_tokens`` is the full-attention chunk size and need not + # be a multiple of this SWA group's (DCP-scaled) block size (e.g. + # Gemma). Fall back to dense: every block is reachable, which never + # drops a block that could serve a hit. + return None + + # Contiguous blocks a hit needs at a boundary (incl. the EAGLE peek). + need = cls._contiguous_blocks_for_hit( + window_size=kv_cache_spec.sliding_window, + block_size=block_size, + use_eagle=use_eagle, + ) + # The matched run's right edge sits on the aligned boundary block when + # EAGLE peeks one block past it (shift=1), otherwise on the last block + # before the boundary (shift=0). + shift = 1 if use_eagle else 0 + + mask = [False] * (end_block - start_block) + + # (1) Segment-boundary tails. ``retention_interval``: + # None -> dense (a tail at every ``alignment_tokens`` boundary); + # 0 -> no dense tails (only the replay boundary below); + # >0 -> a tail once per ``retention_interval``-sized segment. + segment_tokens = ( + alignment_tokens + if retention_interval is None + else (None if retention_interval == 0 else retention_interval) + ) + if segment_tokens is not None: + per_segment = segment_tokens // block_size + if need >= per_segment: + # Every block is reachable; cache them all. + return None + for i in range(start_block, end_block): + if i >= shift and (i - shift) % per_segment >= per_segment - need: + mask[i - start_block] = True + + if final_segment_end_block is not None and not use_eagle: + final_segment_size = final_segment_end_block % per_segment + if final_segment_size: + final_segment_start = final_segment_end_block - final_segment_size + final_tail_start = max( + final_segment_start, final_segment_end_block - need + ) + final_tail_end = min(end_block, final_segment_end_block) + for i in range(max(start_block, final_tail_start), final_tail_end): + mask[i - start_block] = True + + # (2) Reachable-boundary tails: the replay boundary (``num_prompt - 1``, + # capped by ``get_computed_blocks``) and any shared-prefix junction. Both + # land before segments would cover them under sparse retention, so keep + # the ``need``-block tail ending on each boundary explicitly. + if retention_interval is not None: + for boundary_tokens in reachable_boundaries: + aligned = boundary_tokens // alignment_tokens * alignment_tokens + end = aligned // block_size + shift + for j in range(max(start_block, end - need), min(end_block, end)): + mask[j - start_block] = True + + return mask + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + """Get the number of tokens that will be skipped for attention computation. + + For sliding window, this corresponds to the tokens that are prior to + the current sliding window. + + Example: + sliding_window=4, num_computed_tokens=7 + + Tokens: [ 0 1 2 3 4 5 6 7 ] + | ---- computed -----| + ^ next token to be computed + |-----------| sliding window for next token + |--skipped---| + + The current window contains tokens 4~7. Tokens 0~3 will be skipped for + attention computation since they are outside the sliding window. + Thus, get_num_skipped_tokens(7) == 4. + + The trailing edge of the window is extended by ``extra_retained_tokens`` + so that those extra trailing tokens' blocks are retained (but not + attended). This is needed for multi-module spec decoding which can + re-prefill the last num_spec_prefill_tokens - 1 tokens from the end + of the sequence, and thus needs to delay freeing/caching of blocks. + + Args: + num_computed_tokens: The number of tokens that have been computed. + + Returns: + The number of tokens that will be skipped for attention computation. + + """ + return max( + 0, + num_computed_tokens - self.sliding_window + 1 - self.extra_retained_tokens, + ) + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + """NOTE(Chen): The prefix blocks are null blocks for sliding window layers. + So it's not correct to count ref_cnt like FullAttentionManager. Return + 0 here for correctness. Need to support cascade attention + sliding + window in the future. + """ + return 0 + + +class _RingSlotPage: + """One pool block shared by up to ``slots`` requests' rings.""" + + __slots__ = ("block", "free_slots", "live") + + def __init__(self, block: KVCacheBlock, slots: int) -> None: + self.block = block + # Highest first, so pop() hands out the lowest free slot. + self.free_slots = list(range(slots - 1, -1, -1)) + self.live = 0 + + +class RingSlotPool: + """v2.5 side-cache layout: ring blocks as slots of shared pool blocks. + + The QSA ring needs one block per request for the request's lifetime, but + its pages for every layer of a stage are ~16 KB against a 12.9 MB pool + block. This pool takes whole blocks from the main ``BlockPool`` and cuts + each into ``slots_per_block`` slots; a slot's block id is + ``pool_block_id * slots_per_block + slot``, which is exactly the index of + the slot in the worker's ring view (``KVCacheTensor.slots_per_block``). + + The slot blocks it hands out carry ``pool=self``, so every existing free + path -- ``SingleTypeKVCacheManager.free``, the coordinator's deferred + ``pop_blocks_for_free`` and the scheduler's fenced drain, which all call + ``BlockPool.free_blocks`` -- routes them back here, at the time the old + per-request block would have gone back (the deferred free is kept). A pool + block returns to the main pool when its last slot is freed. Rings are never + prefix-cached, so a slot block carries no hash. + """ + + def __init__(self, block_pool: BlockPool, slots_per_block: int) -> None: + assert slots_per_block > 1 + self.block_pool = block_pool + self.slots_per_block = slots_per_block + # Pool block id -> page, in claim order (the oldest page fills first). + self._pages: dict[int, _RingSlotPage] = {} + + @property + def num_pages(self) -> int: + """Pool blocks currently held for rings.""" + return len(self._pages) + + @property + def num_live_slots(self) -> int: + return sum(page.live for page in self._pages.values()) + + def has_free_slot(self) -> bool: + return any(page.free_slots for page in self._pages.values()) + + def get_new_blocks(self, num_blocks: int) -> list[KVCacheBlock]: + out: list[KVCacheBlock] = [] + for _ in range(num_blocks): + page = next((p for p in self._pages.values() if p.free_slots), None) + if page is None: + (pool_block,) = self.block_pool.get_new_blocks(1) + page = _RingSlotPage(pool_block, self.slots_per_block) + self._pages[pool_block.block_id] = page + slot = page.free_slots.pop() + page.live += 1 + out.append( + KVCacheBlock( + page.block.block_id * self.slots_per_block + slot, + ref_cnt=1, + pool=self, # type: ignore[arg-type] + ) + ) + return out + + def free_blocks(self, ordered_blocks) -> None: + released: list[KVCacheBlock] = [] + for block in ordered_blocks: + assert block.pool is self, "not a ring slot of this pool" + block.ref_cnt -= 1 + assert block.ref_cnt == 0, "ring slots have a single owner" + page_id, slot = divmod(block.block_id, self.slots_per_block) + page = self._pages[page_id] + assert slot not in page.free_slots, f"slot {block.block_id} freed twice" + page.free_slots.append(slot) + page.free_slots.sort(reverse=True) + page.live -= 1 + if page.live == 0: + del self._pages[page_id] + released.append(page.block) + if released: + self.block_pool.free_blocks(released) + + +class CircularBufferManager(FullAttentionManager): + """Claims the ring's single block per request; prefix caching disabled.""" + + supports_fine_grained_hash_lookup: ClassVar[bool] = False + + # v2.5 side-cache layout: set by the coordinator for a slot-packed group. + _slot_pool: RingSlotPool | None = None + + def enable_slot_packing(self, slots_per_block: int) -> None: + """Claim ring slots of shared pool blocks instead of whole blocks.""" + assert not self.req_to_blocks, "enable slot packing before any claim" + self._slot_pool = RingSlotPool(self.block_pool, slots_per_block) + + @property + def slot_pool(self) -> RingSlotPool | None: + return self._slot_pool + + if self._slot_pool is not None: + new_blocks = self._slot_pool.get_new_blocks(1) + else: + new_blocks = self.block_pool.get_new_blocks(1) + + def _claim_ring_block( + self, request_id: str, record_for_zeroing: bool = True + ) -> list[KVCacheBlock]: + req_blocks = self.req_to_blocks[request_id] + if req_blocks: + return [] + if self._slot_pool is not None: + new_blocks = self._slot_pool.get_new_blocks(1) + else: + new_blocks = self.block_pool.get_new_blocks(1) + req_blocks.extend(new_blocks) + if self._record_new_block_ids and record_for_zeroing: + self.new_block_ids.extend(block.block_id for block in new_blocks) + # Unconditional -- not gated on needs_kv_cache_zeroing. + self.new_ring_block_ids.extend(block.block_id for block in new_blocks) + return new_blocks + + def take_new_ring_block_ids(self) -> list[int]: + """Drain the ring blocks claimed since the last call, keeping only + blocks a request of this manager still holds, in claim order, once. + + A block claimed and then freed within the same scheduling pass (the + request was preempted after its allocation) may already belong to + another group by the time the worker acts; clearing its tag words + there would write into that group's page, so it is dropped here. + """ + claimed = self.new_ring_block_ids + self.new_ring_block_ids = [] + if not claimed: + return [] + held = { + block.block_id for blocks in self.req_to_blocks.values() for block in blocks + } + out: list[int] = [] + seen: set[int] = set() + for block_id in claimed: + if block_id in held and block_id not in seen: + seen.add(block_id) + out.append(block_id) + return out + + def get_ring_block_ids(self, request_id: str) -> list[int]: + return [block.block_id for block in self.req_to_blocks.get(request_id, ())] + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + prefill_end: int = 0, + ) -> int: + return 0 if self.req_to_blocks.get(request_id) else 1 + + def allocate_new_blocks( + self, request_id: str, num_tokens: int, num_tokens_main_model: int + ) -> list[KVCacheBlock]: + return self._claim_ring_block(request_id) + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + record_for_zeroing: bool = True, + ) -> None: + self._claim_ring_block(request_id, record_for_zeroing) + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + return tuple([] for _ in kv_cache_group_ids), 0 + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + return + + def add_local_computed_blocks( + self, + request_id: str, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + return + + def remove_skipped_blocks( + self, + request_id: str, + processed_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + return + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + return 0 + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + return 0 + + +class KpoolTailManager(CircularBufferManager): + """One-block circular scratch manager for ``KpoolTailSpec``.""" + + +class ChunkedLocalAttentionManager(SingleTypeKVCacheManager): + def __init__(self, kv_cache_spec: ChunkedLocalAttentionSpec, **kwargs) -> None: + super().__init__(kv_cache_spec, **kwargs) + self.attention_chunk_size = kv_cache_spec.attention_chunk_size + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + """For chunked local attention, we need to find the longest cache hit + prefix of the blocks that is not longer than `max_length`. The prefix + should be a common prefix hit for all the kv cache groups in + `kv_cache_group_ids`. If no cache hit is found, return an empty list. + note we mark as computed if the whole block is outside of the local + window, and set the block as null. Examples: + + 1. Attention chunk size of 8, block size of 4, max length of 15 + for next token at 15th (zero-indexed), 8th - 14th tokens are in + the window(needs lookup), 0th - 7th are not in the window, + so they are already marked as computed. We check the complete + block3 (8th - 11th tokens), Assume block 3 is hit, we will return + [null, null, block 3], otherwise, we return [null, null] + + 2. Attention chunk size of 8, block size of 4, max length of 16 + for next token at 16th (zero-indexed), 0th - 15th tokens are not + in the window, so they are already marked as computed. + we return 4 blocks[null, null, null, null] + + Args: + block_hashes: The block hashes of the request. + max_length: The maximum length of the cache hit prefix. + kv_cache_group_ids: The ids of the kv cache groups. + block_pool: The block pool. + kv_cache_spec: The kv cache spec. + drop_eagle_block: Whether to drop the last matched block for EAGLE/MTP. + dcp_world_size: The world size of decode context parallelism. + pcp_world_size: The world size of prefill context parallelism. + alignment_tokens: The returned cache hit length (in tokens) should + be a multiple of this value (in tokens). + + Returns: + A list of cached blocks + + """ + assert isinstance(kv_cache_spec, ChunkedLocalAttentionSpec), ( + "ChunkedLocalAttentionManager can only be used for " + "chunked local attention groups" + ) + assert drop_eagle_block is False, ( + "Hybrid KV cache is not supported for " + "eagle + chunked local attention." + ) + assert dcp_world_size == 1, "DCP not support chunked local attn now." + assert pcp_world_size == 1, "PCP not support chunked local attn now." + assert kv_cache_spec.block_size == alignment_tokens, ( + "KV cache groups with different block sizes are not compatible with " + "chunked local attention now" + ) + block_hashes = resolve_block_hashes( + block_hashes, + block_pool.hash_block_size, + kv_cache_spec.block_size, + supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup, + alignment_tokens=alignment_tokens, + ) + max_num_blocks = max_length // kv_cache_spec.block_size + if max_length > 0: + local_attention_start_idx = ( + max_length + // kv_cache_spec.attention_chunk_size + * kv_cache_spec.attention_chunk_size + ) + else: + local_attention_start_idx = 0 + # we marked blocks out of window as computed + # with null blocks, and blocks inside window based on cache lookup + # result [null] [null] ... [null] [hit block 1 (1st block contain + # last window)] [hit block 2] ... [hit block x] + local_attention_start_block_idx = ( + local_attention_start_idx // kv_cache_spec.block_size + ) + computed_blocks: tuple[list[KVCacheBlock], ...] = tuple( + [block_pool.null_block] * local_attention_start_block_idx + for _ in range(len(kv_cache_group_ids)) + ) + for i in range(local_attention_start_block_idx, max_num_blocks): + block_hash = block_hashes[i] + if cached_block := block_pool.get_cached_block( + block_hash, kv_cache_group_ids + ): + for computed, cached in zip(computed_blocks, cached_block): + computed.append(cached) + else: + break + hit_length = len(computed_blocks[0]) * kv_cache_spec.block_size + return computed_blocks, hit_length + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + """Get the number of tokens that will be skipped for attention computation. + + For chunked local attention, this corresponds to the tokens that are on + the left side of the current chunk. + + Example 1: + chunk size = 8, num_computed_tokens = 13 + Tokens: [ 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15 ] ... + | ----- computed ---------------| + ^^ next token to be computed + |----------------| <-- attention window for + next token + |--- skipped -----| + Output: get_num_skipped_tokens(13) == 8 + + Example 2: + chunk size = 8, num_computed_tokens = 8 + Tokens: [ 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15 ] ... + | --- computed ---| + ^ next token to be computed + |--| <-- attention window for next token + | --- skipped ----| + Output: get_num_skipped_tokens(8) == 8 + + Example 3: + chunk size = 8, num_computed_tokens = 7 + Tokens: [ 0 1 2 3 4 5 6 7 | 8 9 10 11 12 13 14 15 ] ... + |---computed---| + ^ next token to be computed + |-----------------| <-- attention window for next token + no token should be skipped. + Output: get_num_skipped_tokens(7) == 0 + + Args: + num_computed_tokens: The number of tokens that have been computed. + + Returns: + The number of tokens that will be skipped for attention computation. + + """ + num_skipped_tokens = ( + num_computed_tokens // self.attention_chunk_size + ) * self.attention_chunk_size + return num_skipped_tokens + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + """Cascade attention is not supported by chunked local attention.""" + return 0 + + +class MambaManager(SingleTypeKVCacheManager): + supports_fine_grained_hash_lookup: ClassVar[bool] = True + + @property + def has_positionally_stable_blocks(self) -> bool: + # Align-mode Mamba can null interior states and relocate speculative + # blocks in place. Other modes retain positional identity. + return self.mamba_cache_mode != "align" + + def __init__( + self, kv_cache_spec: MambaSpec, block_pool: BlockPool, **kwargs + ) -> None: + super().__init__(kv_cache_spec, block_pool, **kwargs) + self.mamba_cache_mode = kv_cache_spec.mamba_cache_mode + self.num_speculative_blocks: int = kv_cache_spec.num_speculative_blocks + self.has_prefill_checkpoint_blocks = ( + self.mamba_cache_mode == "align" + and kv_cache_spec.num_prefill_checkpoint_blocks > 0 + ) + # Mamba checkpoints follow Eagle's global replay boundary. + self.drop_eagle_checkpoint_block = False + self.cached_blocks_this_step: set[BlockHashWithGroupId] = set() + if self.mamba_cache_mode == "align": + # Mapping from request ID to the index of the block + # allocated in the previous step + self.last_state_block_idx: dict[str, int] = {} + self._num_retired_blocks: dict[str, int] = {} + # The set of the requests that have been allocated blocks + self._allocated_block_reqs: set[str] = set() + # checkpoint position and reserved block index for the current + # allocation. + self._checkpoints: dict[str, tuple[int, int]] = {} + # Requests that registered their own last-prompt-boundary partial + # tail (producers). A later CoW hands its private copy to the + # connector; a request that finishes first hands off this table + # source directly. + self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {} + # v2.5. In align mode a state block leaves the block table (it is + # nulled and released) as soon as the request computes one block past + # it -- `_remove_blocks_in_range` / `remove_skipped_blocks`. When that + # block carries a prefix-cache hash it is the request's GDN checkpoint + # (the replay boundary under retention interval 0), and releasing it + # mid-request puts it in the free queue AHEAD of the request's own + # attention blocks, which stay referenced until finish. Concurrent + # prefills then pop it first; since the hybrid hit is the shortest + # group's, that single block zeroes the next turn's reuse. With + # retention on, the newest hashed retired state stays referenced here + # (at most one per request per group) and is released only when the + # request is freed, after its attention blocks (coordinator.free). + self._retain_checkpoint: bool = ( + self.mamba_cache_mode == "align" + and self.enable_caching + and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT + ) + # {request_id: (logical block index, block)} + self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {} + if self.mamba_cache_mode == "align" and self.enable_caching: + # The A/B evidence line: boot logs must show which arm ran. + logger.info_once( + "Mamba align prefix-checkpoint retention: %s " + "(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)", + "on" if self._retain_checkpoint else "off", + ) + + def _retire_state_blocks( + self, request_id: str, retired: list[tuple[int, KVCacheBlock]] + ) -> None: + """Release align-mode state blocks the request no longer reads. + + ``retired`` holds (logical index, block) pairs already nulled out of + the block table, in the eviction order the caller wants. With + retention on, the newest hashed one is kept referenced as the request's + checkpoint instead, and the checkpoint it supersedes (if any) is + released now -- so the request never holds more than one. + """ + if self._retain_checkpoint: + newest: tuple[int, KVCacheBlock] | None = None + for idx, block in retired: + if block.block_hash is not None and ( + newest is None or idx > newest[0] + ): + newest = (idx, block) + held = self._retained_checkpoints.get(request_id) + if newest is not None and (held is None or newest[0] > held[0]): + self._retained_checkpoints[request_id] = newest + retired = [entry for entry in retired if entry[1] is not newest[1]] + if held is not None: + # Superseded: it ages from here like any cached free block. + retired.append(held) + if retired: + self.block_pool.free_blocks([block for _, block in retired]) + + def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]: + if self.mamba_cache_mode != "align": + return [] + held = self._retained_checkpoints.pop(request_id, None) + return [] if held is None else [held[1]] + + def get_num_retained_checkpoint_blocks_to_reserve( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock] = (), + ) -> int: + # A request that already holds a checkpoint only ever swaps it. One + # that does not will hold one once a hashed state retires, which needs + # at least one state block behind the running one. + if ( + not self._retain_checkpoint + or request_id in self._retained_checkpoints + or num_tokens <= self.block_size + ): + return 0 + # A prefix hit already brings the block that becomes the held + # checkpoint -- the hit state, counted by get_num_blocks_to_allocate as + # an evictable computed block (or already referenced elsewhere) -- and + # the same holds for a hashed state still in the request's table. + # Reserving another one double-counts every hit admission by one block + # per group, which only delays the follow-up turns the fix is for. + if any(not blk.is_null for blk in new_computed_blocks): + return 0 + if any( + not blk.is_null and blk.block_hash is not None + for blk in self.req_to_blocks.get(request_id, ()) + ): + return 0 + return 1 + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + assert isinstance(kv_cache_spec, MambaSpec), ( + "MambaManager can only be used for mamba groups" + ) + assert dcp_world_size == 1, "DCP not support mamba now." + assert pcp_world_size == 1, "PCP not support mamba now." + block_hashes = resolve_block_hashes( + block_hashes, + block_pool.hash_block_size, + kv_cache_spec.block_size, + supports_fine_grained_hash_lookup=cls.supports_fine_grained_hash_lookup, + alignment_tokens=alignment_tokens, + ) + computed_blocks: tuple[list[KVCacheBlock], ...] = tuple( + [] for _ in range(len(kv_cache_group_ids)) + ) + hit_length = 0 + + block_size = kv_cache_spec.block_size + if alignment_tokens < block_size and block_size % alignment_tokens == 0: + # list or lazy BlobBlockHashes view + assert isinstance(block_hashes, Sequence) + hash_block_size = alignment_tokens + scale_factor = block_size // hash_block_size + max_num_partial_units = min( + max_length // hash_block_size, len(block_hashes) + ) + for fine_idx in range(max_num_partial_units - 1, -1, -1): + num_tokens = (fine_idx + 1) * hash_block_size + block_hash = block_hashes[fine_idx] + if cached_block := block_pool.get_cached_block( + block_hash, kv_cache_group_ids + ): + block_idx = fine_idx // scale_factor + for computed, cached in zip(computed_blocks, cached_block): + computed.extend([block_pool.null_block] * block_idx) + computed.append(cached) + hit_length = num_tokens + break + return computed_blocks, hit_length + + max_num_blocks = max_length // block_size + if drop_eagle_block and max_num_blocks > 0: + # EAGLE/MTP: drop the final matched page. Its recurrent-state snapshot + # may have been taken over draft tokens that verification later rejects, + # so the state must be restored from the previous page boundary and + # recomputed forward. FullAttentionManager (see find_longest_cache_hit + # above) pops the last block; Mamba keeps only the rightmost real block + # with the prefix null-padded, so a literal pop would delete the state + # block itself -- lower the search ceiling by one page instead. + max_num_blocks -= 1 + # Search from right to left and early stop when a match is found. + for i in range(max_num_blocks - 1, -1, -1): + if cached_block := block_pool.get_cached_block( + block_hashes[i], kv_cache_group_ids + ): + # When enable Mamba prefix caching, `block_size` will be aligned + # across full attention layers and Mamba layers to ensure the + # prefix hit length aligned at block + if ( + block_size != alignment_tokens # Faster for common case. + and (i + 1) * block_size % alignment_tokens != 0 + ): + continue + for computed, cached in zip(computed_blocks, cached_block): + # the hit length logic later assumes: + # hit_length = len(hit_blocks_other_attn[0]) + # * self.other_block_size + # so we insert dummy blocks at the beginning: + computed.extend([block_pool.null_block] * i) + computed.append(cached) + hit_length = (i + 1) * block_size + break # we just need the last match - early stopping + + return computed_blocks, hit_length + + @classmethod + def reachable_block_mask( + cls, + start_block: int, + end_block: int, + alignment_tokens: int | None, + kv_cache_spec: KVCacheSpec, + use_eagle: bool, + retention_interval: int | None = None, + reachable_boundaries: Sequence[int] = (), + dcp_world_size: int = 1, + final_segment_end_block: int | None = None, + ) -> list[bool] | None: + """Sparse Mamba state-snapshot retention. + + ``retention_interval``: + + ``None`` -> dense (cache every block; default, unchanged behavior) + ``0`` -> keep only the ``reachable_boundaries`` states + ``> 0`` -> keep one state per ``retention_interval``-sized segment + + ``reachable_boundaries`` are proven reuse points (the replay boundary and + any cross-request shared-prefix junction, Marconi-style APC); their + boundary state is always kept so sparse retention does not defeat reuse. + """ + if retention_interval is None or alignment_tokens is None: + # Dense caching (default) or no alignment constraint imposed. + return None + assert isinstance(kv_cache_spec, MambaSpec) + block_size = kv_cache_spec.block_size + mask = [False] * (end_block - start_block) + + # (1) Segment-boundary states. A Mamba hit needs exactly the single + # state block ending on the boundary (no window, and draft models have + # no mamba layers, so no eagle shift). Block ``i`` ends at token + # ``(i + 1) * block_size``. + segment_tokens = None if retention_interval == 0 else retention_interval + if segment_tokens is not None: + per_segment = segment_tokens // block_size + if per_segment <= 1: + # Interval at/below the block size: every block is a boundary. + return None + first_boundary = ( + start_block + per_segment + ) // per_segment * per_segment - 1 + for i in range(first_boundary - start_block, len(mask), per_segment): + mask[i] = True + + # (2) Reachable-boundary states: the replay boundary (``num_prompt - 1``, + # capped by ``get_computed_blocks``) and any shared-prefix junction, both + # of which segments would otherwise skip under sparse retention. A Mamba + # hit needs exactly the single state block ending on the boundary. + for boundary_tokens in reachable_boundaries: + aligned = boundary_tokens // alignment_tokens * alignment_tokens + boundary_block = aligned // block_size - 1 + if start_block <= boundary_block < end_block: + mask[boundary_block - start_block] = True + + return mask + + def _remove_blocks_in_range( + self, request_id: str, first_block: int, last_block: int + ) -> None: + if self.mamba_cache_mode != "align": + return super()._remove_blocks_in_range(request_id, first_block, last_block) + blocks = self.req_to_blocks.get(request_id, []) + first_block = max(first_block, self._num_retired_blocks.get(request_id, 0)) + last_block = min(last_block, len(blocks)) + if first_block >= last_block: + return + freed: list[tuple[int, KVCacheBlock]] = [] + # Mamba prefill leaves null gaps between states awaiting retirement. + for i in range(last_block - 1, first_block - 1, -1): + if blocks[i].is_null: + continue + freed.append((i, blocks[i])) + blocks[i] = self._null_block + self._retire_state_blocks(request_id, freed) + self._num_retired_blocks[request_id] = last_block + + def remove_skipped_blocks( + self, + request_id: str, + processed_computed_tokens: int, + num_prompt_tokens: int | None = None, + ) -> None: + assert isinstance(self.kv_cache_spec, MambaSpec) + + super().remove_skipped_blocks( + request_id, processed_computed_tokens, num_prompt_tokens + ) + if self.mamba_cache_mode == "align": + # `last_state_block_idx` refers to the block index allocated two steps ago. + # The block allocated in the previous step is used to copy Mamba states + # into the block allocated in the current step; the earlier block is + # no longer needed and should be freed here. + last_state_block_idx = self.last_state_block_idx.get(request_id) + # Blocks allocated during prefill may be non-contiguous. Use + # `last_state_block_idx` to free the appropriate block and replace it + # with a null block. + if ( + last_state_block_idx is not None + and last_state_block_idx + < cdiv(processed_computed_tokens, self.block_size) - 1 + ): + blocks = self.req_to_blocks[request_id] + if blocks[last_state_block_idx] != self._null_block: + block = blocks[last_state_block_idx] + blocks[last_state_block_idx] = self._null_block + self._retire_state_blocks( + request_id, [(last_state_block_idx, block)] + ) + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + """Cascade attention is not supported by mamba""" + return 0 + + def _needs_internal_checkpoint( + self, + request_id: str, + query_start: int, + query_end: int, + checkpoint_position: int, + prefill_end: int, + ) -> bool: + assert isinstance(self.kv_cache_spec, MambaSpec) + checkpoint_idx = cdiv(query_end, self.block_size) - 2 + blocks = self.req_to_blocks[request_id] + return ( + self.has_prefill_checkpoint_blocks + and query_end >= prefill_end + and is_mamba_prefill_checkpoint_valid( + query_start=query_start, + query_end=query_end, + checkpoint_position=checkpoint_position, + hash_block_size=self.block_pool.hash_block_size, + mamba_block_size=self.block_size, + checkpoint_alignment=(self.kv_cache_spec.prefill_checkpoint_alignment), + ) + and checkpoint_idx >= 0 + and ( + checkpoint_idx >= len(blocks) + or blocks[checkpoint_idx].is_null + or ( + request_id in self._allocated_block_reqs + and checkpoint_idx >= len(blocks) - self.num_speculative_blocks + ) + ) + ) + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + prefill_end: int = 0, + ) -> int: + assert isinstance(self.kv_cache_spec, MambaSpec) + if ( + len(new_computed_blocks) > 0 + and new_computed_blocks[-1].block_hash in self.cached_blocks_this_step + ): + # Mamba can't rely on blocks generated by other requests in the current step + # To put it in the next step, we return num_gpu_blocks + 1 so + # that kv_cache_manager will think there is no enough blocks to allocate now + # and don't schedule it in the current step. + return self.block_pool.num_gpu_blocks + 1 + if self.mamba_cache_mode != "align": + # Allocate extra `num_speculative_blocks` blocks for + # speculative decoding (MTP/EAGLE) with linear attention. + if self.num_speculative_blocks > 0: + num_tokens += ( + self.kv_cache_spec.block_size * self.num_speculative_blocks + ) + return super().get_num_blocks_to_allocate( + request_id, + num_tokens, + new_computed_blocks, + total_computed_tokens, + num_local_computed_tokens, + num_tokens_main_model, + apply_admission_cap=apply_admission_cap, + prefill_end=prefill_end, + ) + else: + # We don't allocate blocks for lookahead tokens in align mode, because if + # x * block_size tokens are scheduled, num_tokens is + # x * block_size + num_lookahead_tokens and breaks the alignment. + # We can ignore lookahead tokens because current draft models don't have + # mamba layers. + num_tokens = num_tokens_main_model + + # NOTE(tdouble): this is an over-estimate of how many blocks we need because + # num_tokens can include draft tokens that will later be rejected. + num_required_blocks = ( + cdiv(num_tokens, self.block_size) + self.num_speculative_blocks + ) + num_new_blocks = ( + num_required_blocks + - len(new_computed_blocks) + - len(self.req_to_blocks[request_id]) + ) + has_partial_hit = ( + self._has_partial_local_hit( + new_computed_blocks, num_local_computed_tokens + ) + or request_id in self._partial_hit_reqs + ) + if has_partial_hit: + num_new_blocks = max(num_new_blocks, 0) + 1 + # Keyed on the chunk end like the worker's + # `compute_mamba_prefill_checkpoints`, which writes the state there. + # The helper returns a boundary strictly below its input, so every + # chunk has one; under sparse retention only the prefill-end chunk's + # is kept, so `prefill_end` skips reserving blocks for the others. + checkpoint_position = get_mamba_prefill_checkpoint_position( + num_tokens, + self.block_pool.hash_block_size, + self.drop_eagle_checkpoint_block, + ) + if not self._needs_internal_checkpoint( + request_id, + total_computed_tokens, + num_tokens, + checkpoint_position, + prefill_end, + ): + checkpoint_position = 0 + checkpoint_block = int(checkpoint_position > 0) + if not apply_admission_cap: + if checkpoint_position > 0: + checkpoint_idx = cdiv(num_tokens, self.block_size) - 2 + self._checkpoints[request_id] = ( + checkpoint_position, + checkpoint_idx, + ) + else: + self._checkpoints.pop(request_id, None) + if num_new_blocks > 0: + blocks_allocated = request_id in self._allocated_block_reqs + physical_block_cap = 1 + int(has_partial_hit) + checkpoint_block + if not blocks_allocated or checkpoint_block: + physical_block_cap += self.num_speculative_blocks + num_new_blocks = min(num_new_blocks, physical_block_cap) + + num_evictable_computed_blocks = self._get_num_evictable_blocks( + new_computed_blocks + ) + return num_new_blocks + num_evictable_computed_blocks + + def allocate_new_blocks( + self, request_id: str, num_tokens: int, num_tokens_main_model: int + ) -> list[KVCacheBlock]: + assert isinstance(self.kv_cache_spec, MambaSpec) + if self.mamba_cache_mode != "align": + # Allocate extra `num_speculative_blocks` blocks for + # speculative decoding (MTP/EAGLE) with linear attention. + if self.num_speculative_blocks > 0: + num_tokens += self.block_size * self.num_speculative_blocks + return super().allocate_new_blocks( + request_id, num_tokens, num_tokens_main_model + ) + else: + # We don't allocate blocks for lookahead tokens in align mode, because if + # x * block_size tokens are scheduled, num_tokens is + # x * block_size + num_lookahead_tokens and breaks the alignment. + # We can ignore lookahead tokens because current draft models don't have + # mamba layers. + num_tokens = num_tokens_main_model + req_blocks: list[KVCacheBlock] = self.req_to_blocks[request_id] + # NOTE(tdouble): this is an over-estimate of how many blocks we need because + # num_tokens can include draft tokens that will later be rejected. + num_required_blocks = ( + cdiv(num_tokens, self.block_size) + self.num_speculative_blocks + ) + checkpoint_block = int(request_id in self._checkpoints) + partial_hit = self._partial_hit_reqs.get(request_id) + has_partial_hit = partial_hit is not None + # `num_required_blocks` might be less than `len(req_blocks)` if blocks are + # over-allocated at last round. + if ( + num_required_blocks <= len(req_blocks) + and not has_partial_hit + and not checkpoint_block + ): + self._allocated_block_reqs.add(request_id) + return [] + else: + prev_block_len = len(req_blocks) + blocks_allocated = request_id in self._allocated_block_reqs + # Record the last state block + if blocks_allocated: + # We always save the running state at the last + # (1 + num_speculative_blocks) block + self.last_state_block_idx[request_id] = ( + prev_block_len - 1 - self.num_speculative_blocks + ) + elif prev_block_len > 0: + # When a new request hits the prefix cache, the last block + # saves the hit state. + self.last_state_block_idx[request_id] = prev_block_len - 1 + + num_skipped_blocks = ( + num_required_blocks - self.num_speculative_blocks - 1 + ) + # null blocks + if prev_block_len < num_skipped_blocks: + # minus the internal checkpoint block + # so we don't set null for that block + null_end = num_skipped_blocks - checkpoint_block + req_blocks.extend( + [self._null_block for _ in range(prev_block_len, null_end)] + ) + + if blocks_allocated and not checkpoint_block: + # Relocate exclusively owned speculative scratch blocks. + for block_idx in range( + prev_block_len - self.num_speculative_blocks, prev_block_len + ): + if block_idx < num_skipped_blocks: + self._relocate_speculative_block(req_blocks, block_idx) + else: + break + num_new_blocks = max(num_required_blocks - len(req_blocks), 0) + if has_partial_hit: + num_new_blocks = max(num_new_blocks, 0) + 1 + max_new_blocks = 1 + int(has_partial_hit) + checkpoint_block + if not blocks_allocated or checkpoint_block: + max_new_blocks += self.num_speculative_blocks + assert num_new_blocks <= max_new_blocks + new_blocks = self.block_pool.get_new_blocks(num_new_blocks) + returned_blocks = req_blocks[prev_block_len:] + if partial_hit is not None: + block_idx, source_block = partial_hit + cow_block = new_blocks[0] + new_blocks = new_blocks[1:] + if blocks_allocated: + # The worker block table of a running request is + # append-only, so the request must stay on + # source_block. Move the cache entry to cow_block + # instead; the queued copy fills it before forward + # overwrites source_block. + assert req_blocks[block_idx] is source_block + self.block_pool.move_block_hashes(source_block, cow_block) + self._pending_cow_copies.append((source_block, cow_block)) + source_block.ref_cnt += 1 + producer_tail = self._producer_partial_tail_reqs.pop( + request_id, None + ) + if producer_tail is not None: + marker_block, boundary_tokens = producer_tail + assert marker_block is source_block + # This CoW preserved a producer's own boundary + # state in cow_block; hand it to the connector for + # partial-tail offload once the copy has run. + self._pending_boundary_state_offloads.append( + ( + request_id, + self.kv_cache_group_id, + cow_block, + boundary_tokens, + ) + ) + if cow_block.block_hash is not None: + # The moved entry is only filled by this step's + # copy, so defer same-step hits on it. + self.cached_blocks_this_step.add(cow_block.block_hash) + else: + self._apply_cow(request_id, block_idx, source_block, cow_block) + returned_blocks = [cow_block] + returned_blocks + req_blocks.extend(new_blocks) + self._allocated_block_reqs.add(request_id) + self._partial_hit_reqs.pop(request_id, None) + returned_blocks.extend(new_blocks) + return returned_blocks + + def _relocate_speculative_block( + self, req_blocks: list[KVCacheBlock], block_idx: int + ) -> None: + block = req_blocks[block_idx] + assert self.block_pool.is_block_writable(block), ( + "Speculative Mamba blocks must be exclusively owned and unhashed " + "before relocation" + ) + req_blocks.append(block) + req_blocks[block_idx] = self._null_block + + def finalize_partial_tail_offload( + self, + request_id: str, + num_computed_tokens: int, + num_in_flight_tokens: int, + ) -> tuple[int, KVCacheBlock, int] | None: + if self.mamba_cache_mode != "align": + return None + producer_tail = self._producer_partial_tail_reqs.pop(request_id, None) + if producer_tail is None: + return None + source_block, boundary_tokens = producer_tail + if num_in_flight_tokens != 0 or num_computed_tokens != boundary_tokens: + return None + return self.kv_cache_group_id, source_block, boundary_tokens + + def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: + if self.mamba_cache_mode == "align": + self._allocated_block_reqs.discard(request_id) + self.last_state_block_idx.pop(request_id, None) + self._num_retired_blocks.pop(request_id, None) + self._checkpoints.pop(request_id, None) + self._producer_partial_tail_reqs.pop(request_id, None) + # An offer is only guaranteed to hold committed bytes until the end + # of the pass that made it. This request's blocks are going back to + # the pool now, so drop its not-yet-offered hand-offs rather than + # let a connector claim a block another request may already have + # been handed. + self._pending_boundary_state_offloads = [ + entry + for entry in self._pending_boundary_state_offloads + if entry[0] != request_id + ] + # The coordinator pops the retained checkpoint first and frees it + # after every group. A direct caller still gets it back, first in + # allocation order, so the caller's reversed free releases it last. + retained = self.pop_retained_checkpoints(request_id) + if retained: + return retained + super().pop_blocks_for_free(request_id) + return super().pop_blocks_for_free(request_id) + + def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: + """Get the number of tokens whose mamba state are not needed anymore. Mamba only + need to keep the state of the last computed token, so we return + num_computed_tokens - 1. + """ + return num_computed_tokens - 1 + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + num_cached_blocks_before = self.num_cached_block.get(request.request_id, 0) + super().cache_blocks( + request, + num_tokens, + retention_interval=retention_interval, + replay_boundaries=replay_boundaries, + ) + num_cached_blocks_after = self.num_cached_block.get(request.request_id, 0) + if self.mamba_cache_mode == "align": + partial_hash = self._cache_partial_tail_block( + request, num_tokens, retention_interval=retention_interval + ) + if partial_hash is not None: + self.cached_blocks_this_step.add(partial_hash) + if num_cached_blocks_after > num_cached_blocks_before: + blocks = self.req_to_blocks[request.request_id] + for idx in range(num_cached_blocks_before, num_cached_blocks_after): + block = blocks[idx] + # Skip null blocks (align-mode skipped states) and blocks that + # were not cached this step — with sparse retention + # (reachable_block_mask) the intermediate state snapshots carry + # no hash and must not be recorded as cached-this-step. + if block.is_null or block.block_hash is None: + continue + self.cached_blocks_this_step.add(block.block_hash) + if self.mamba_cache_mode == "align": + assert block.block_hash_num_tokens is not None + # Offer every retained boundary with its exact block. + # The connector filters against its save window, which may + # extend past the original prompt during resumed prefill. + self._pending_boundary_state_offloads.append( + ( + request.request_id, + self.kv_cache_group_id, + block, + block.block_hash_num_tokens, + ) + ) + + def new_step_starts(self) -> None: + self.cached_blocks_this_step.clear() + + def _cache_partial_tail_block( + self, + request: Request, + num_tokens: int, + retention_interval: int | None, + ) -> BlockHashWithGroupId | None: + hash_block_size = self.block_pool.hash_block_size + # Re-key the reserved block at its exported checkpoint boundary. + checkpoint = self._checkpoints.get(request.request_id) + if checkpoint is not None: + checkpoint_position, checkpoint_idx = checkpoint + blocks = self.req_to_blocks[request.request_id] + assert 0 <= checkpoint_idx < len(blocks) + checkpoint_block = blocks[checkpoint_idx] + # The prompt's final chunk caches only up to its last full block, + # so ``num_tokens < num_prompt_tokens`` holds there too; its + # checkpoint is the prompt's reusable boundary, not a transient one. + is_prompt_checkpoint = ( + checkpoint_position + == get_mamba_prefill_checkpoint_position( + request.num_prompt_tokens, + hash_block_size, + self.drop_eagle_checkpoint_block, + ) + ) + if ( + retention_interval == 0 + and num_tokens < request.num_prompt_tokens + and not is_prompt_checkpoint + and checkpoint_position != request.shared_prefix_boundary + ): + # retention_interval == 0 keeps this transient checkpoint + # request-local. The slot may carry a hash from this step's + # full-block pass; that must go too, since the checkpoint + # state is about to overwrite the block. + self.block_pool._maybe_evict_cached_block(checkpoint_block) + return None + if checkpoint_block.block_hash_num_tokens == checkpoint_position: + return None + return self.block_pool.cache_partial_block( + request=request, + block=checkpoint_block, + num_tokens=checkpoint_position, + kv_cache_group_id=self.kv_cache_group_id, + block_size=self.block_size, + replace_existing_hashes=True, + ) + if self.block_size == hash_block_size: + return None + if num_tokens % self.block_size == 0: + return None + if num_tokens % hash_block_size != 0: + return None + latest_prompt_hash_boundary = ( + request.num_prompt_tokens // hash_block_size + ) * hash_block_size + if self.drop_eagle_checkpoint_block: + # Eagle groups match one hash unit past the candidate and drop it, + # so register the tail one unit lower. + latest_prompt_hash_boundary = max( + latest_prompt_hash_boundary - hash_block_size, 0 + ) + # The junction is the other position a sibling resumes at: where one was + # observed to stop, and where the scheduler already ends a chunk. Bounded + # to the prompt chunk being computed -- during decode the target is the + # running state block, mutated in place, which equals what its key + # promises only after that step's forward. + if num_tokens != latest_prompt_hash_boundary and not ( + self.shared_prefix_checkpoint + and num_tokens == request.shared_prefix_boundary + and request.num_computed_tokens < num_tokens <= request.num_prompt_tokens + ): + return None + + block_idx = num_tokens // self.block_size + blocks = self.req_to_blocks[request.request_id] + if block_idx >= len(blocks): + return None + source_block = blocks[block_idx] + if source_block.is_null: + return None + + partial_hash = self.block_pool.cache_partial_block( + request=request, + block=source_block, + num_tokens=num_tokens, + kv_cache_group_id=self.kv_cache_group_id, + block_size=self.block_size, + ) + if partial_hash is not None: + self._partial_hit_reqs[request.request_id] = (block_idx, source_block) + self.num_cached_block[request.request_id] = block_idx + # Producer of this partial tail: the boundary state currently lives + # in ``source_block`` but the next step's forward overwrites it. The + # upcoming CoW copies it into a durable cow_block; record the req so + # allocate_new_blocks hands that block to the connector for offload. + self._producer_partial_tail_reqs[request.request_id] = ( + source_block, + num_tokens, + ) + return partial_hash + + +class CrossAttentionManager(SingleTypeKVCacheManager): + """Manager for cross-attention KV cache in encoder-decoder models.""" + + def add_local_computed_blocks( + self, + request_id: str, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + # We do not cache blocks for cross-attention to be shared between + # requests, so `new_computed_blocks` should always be empty. + assert len(new_computed_blocks) == 0 + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + record_for_zeroing: bool = True, + ) -> None: + # Cross-attention does not use prefix caching / external KV loads. + return + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + # We do not cache blocks for cross-attention to be shared between + # requests, so this method is not relevant. + raise ValueError("Should not be called as prefix caching is disabled.") + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + # Cross-attention blocks contain request-specific encoder states + # and are not shared between different requests + return 0 + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + assert isinstance(kv_cache_spec, CrossAttentionSpec), ( + "CrossAttentionManager can only be used for cross-attention groups" + ) + # Cross-attention does not benefit from prefix caching since: + # 1. Encoder states are unique per request (different audio/image + # inputs) + # 2. Encoder states are computed once per request, not incrementally + # 3. No reusable prefix exists between different multimodal inputs + # Return empty blocks to indicate no cache hits + raise NotImplementedError("CrossAttentionManager does not support caching") + + +class SinkFullAttentionManager(FullAttentionManager): + def __init__( + self, + kv_cache_spec: SinkFullAttentionSpec, + block_pool: BlockPool, + enable_caching: bool, + kv_cache_group_id: int, + scheduler_block_size: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ): + super().__init__( + kv_cache_spec=kv_cache_spec, + block_pool=block_pool, + enable_caching=enable_caching, + kv_cache_group_id=kv_cache_group_id, + scheduler_block_size=scheduler_block_size, + dcp_world_size=dcp_world_size, + pcp_world_size=pcp_world_size, + ) + sink_len = kv_cache_spec.sink_len + assert sink_len is not None and sink_len > 0 and sink_len % self.block_size == 0 + + +class HiSparseSourceManager(FullAttentionManager): + """Host-tier manager with a private pool; publishes hashes once durable. + + Every page gets a host block: allocation fails when the host pool cannot + back it, so each GPU-resident page can be written back and later reclaimed. + """ + + coordinator: "HiSparseCoordinator | None" = None + # The host tier keeps prefixes the device groups have already lost. + retains_longer_hit = True + + @property + def records_new_block_ids(self) -> bool: + return False + + def take_new_block_ids(self) -> list[int]: + self.new_block_ids = [] + return [] + + def bind_host_pool(self, num_blocks: int) -> None: + """Replace the device pool this group was built with by a host one.""" + device_pool = self.block_pool + self.block_pool = SharedEventQueueBlockPool( + num_gpu_blocks=num_blocks, + enable_caching=self.enable_caching and self.kv_cache_spec.prefix_cacheable, + hash_block_size=device_pool.hash_block_size, + enable_kv_cache_events=device_pool.enable_kv_cache_events, + metrics_collector=None, + medium=MEDIUM_CPU, + event_owner=device_pool, + ) + self._null_block = self.block_pool.null_block + + def take_pending_cow_copies(self) -> list[tuple[KVCacheBlock, KVCacheBlock]]: + """Host copies never reach the worker's generic block-copy path.""" + return [] + + def take_host_cow_copies(self) -> list[tuple[KVCacheBlock, KVCacheBlock]]: + """Drain host copies for the coordinator to hand to the connector.""" + copies = self._pending_cow_copies + self._pending_cow_copies = [] + return copies + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + prefill_end: int = 0, + ) -> int: + required = super().get_num_blocks_to_allocate( + request_id, + num_tokens, + new_computed_blocks, + total_computed_tokens, + num_local_computed_tokens, + num_tokens_main_model, + ) + # Host blocks come from a private pool, so they never count against + # the GPU pool. Use the same admission sentinel as Mamba instead. + if required > self.block_pool.get_num_free_blocks(): + assert self.coordinator is not None + assert self.coordinator.gpu_pool is not None + return self.coordinator.gpu_pool.num_gpu_blocks + 1 + return 0 + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + record_for_zeroing: bool = True, + ) -> None: + if num_external_computed_tokens <= 0: + return + # The connector writes these pages; only a successful receive makes + # them readable, not advancing the request's computed-token count. + assert self.coordinator is not None + self.coordinator.record_pending_host_import( + request_id, num_local_computed_tokens + num_external_computed_tokens + ) + super().allocate_external_computed_blocks( + request_id, + num_local_computed_tokens, + num_external_computed_tokens, + record_for_zeroing, + ) + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + assert self.coordinator is not None + self.coordinator.publish_when_ready( + request, + num_tokens, + retention_interval, + replay_boundaries=replay_boundaries, + ) + + def publish_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + super().cache_blocks( + request, + num_tokens, + retention_interval=retention_interval, + replay_boundaries=replay_boundaries, + ) + + def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: + assert self.coordinator is not None + self.coordinator.free(request_id) + return super().pop_blocks_for_free(request_id) + + +class _HiSparseAuxiliaryManager(SingleTypeKVCacheManager): + """Base for ephemeral groups whose host source owns prefix caching.""" + + coordinator: "HiSparseCoordinator | None" = None + + def cache_blocks( + self, + request: Request, + num_tokens: int, + retention_interval: int | None = None, + *, + replay_boundaries: Sequence[int], + ) -> None: + return None + + def get_num_common_prefix_blocks(self, running_request_id: str) -> int: + return 0 + + @classmethod + def find_longest_cache_hit( + cls, + block_hashes: BlockHashList, + max_length: int, + kv_cache_group_ids: list[int], + block_pool: BlockPool, + kv_cache_spec: KVCacheSpec, + drop_eagle_block: bool, + alignment_tokens: int, + dcp_world_size: int = 1, + pcp_world_size: int = 1, + ) -> tuple[tuple[list[KVCacheBlock], ...], int]: + return tuple([] for _ in kv_cache_group_ids), 0 + + +class HiSparseHotManager(_HiSparseAuxiliaryManager): + """Allocate a hot region only after a request acquires CPU-only history.""" + + def __init__(self, kv_cache_spec: HiSparseHotSpec, **kwargs) -> None: + super().__init__(kv_cache_spec, **kwargs) + self.blocks_per_request = kv_cache_spec.blocks_per_request + self.hot_required: set[str] = set() + + def require_hot(self, request_id: str) -> None: + self.hot_required.add(request_id) + + def has_hot(self, request_id: str) -> bool: + return len(self.req_to_blocks.get(request_id, ())) == self.blocks_per_request + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + prefill_end: int = 0, + ) -> int: + assert not new_computed_blocks + # A hot region is needed to read host-backed history: an external + # import, a new request resuming a host prefix, or one already asked + # to transition. Running requests keep their earlier answer. + host_import = total_computed_tokens > num_local_computed_tokens + resumes_host_prefix = ( + num_local_computed_tokens > 0 and request_id not in self.num_cached_block + ) + if host_import or resumes_host_prefix or request_id in self.hot_required: + return self.get_num_required_blocks(request_id) + return 0 + + def get_num_required_blocks(self, request_id: str) -> int: + return max( + self.blocks_per_request - len(self.req_to_blocks.get(request_id, ())), + 0, + ) + + def add_local_computed_blocks( + self, + request_id: str, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + assert not new_computed_blocks + self.num_cached_block[request_id] = 0 + assert self.coordinator is not None + if num_local_computed_tokens > 0 or not self.coordinator.resident_managers: + self.require_hot(request_id) + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + record_for_zeroing: bool = True, + ) -> None: + self.require_hot(request_id) + + def allocate_new_blocks( + self, request_id: str, num_tokens: int, num_tokens_main_model: int + ) -> list[KVCacheBlock]: + # Cold admissions can bypass add_local_computed_blocks. + self.num_cached_block[request_id] = 0 + if request_id not in self.hot_required: + return [] + req_blocks = self.req_to_blocks[request_id] + num_new_blocks = self.blocks_per_request - len(req_blocks) + if num_new_blocks <= 0: + return [] + new_blocks = self.block_pool.get_new_blocks(num_new_blocks) + req_blocks.extend(new_blocks) + return new_blocks + + def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: + self.hot_required.discard(request_id) + return super().pop_blocks_for_free(request_id) + + +class HiSparseResidentManager(_HiSparseAuxiliaryManager): + """Track GPU-resident pages for otherwise host-backed KV.""" + + max_admission_blocks_per_request: int + + def get_num_blocks_to_allocate( + self, + request_id: str, + num_tokens: int, + new_computed_blocks: Sequence[KVCacheBlock], + total_computed_tokens: int, + num_local_computed_tokens: int, + num_tokens_main_model: int, + apply_admission_cap: bool = False, + prefill_end: int = 0, + ) -> int: + del num_tokens_main_model + assert not new_computed_blocks + if total_computed_tokens > num_local_computed_tokens: + # Keep the last imported page writable, including the last-token + # replay on a full prompt hit. Only earlier pages stay host-only. + imported_pages = cdiv(total_computed_tokens, self.block_size) + existing = len(self.req_to_blocks.get(request_id, ())) + needs_tail = self.get_resident_page(request_id, imported_pages - 1) is None + return int(needs_tail) + max( + cdiv(num_tokens, self.block_size) - max(imported_pages, existing), 0 + ) + existing = len(self.req_to_blocks.get(request_id, ())) + host_pages = cdiv(num_local_computed_tokens, self.block_size) + required = cdiv(num_tokens, self.block_size) + if apply_admission_cap: + required = min(required, self.max_admission_blocks_per_request) + return max(required - max(existing, host_pages), 0) + + def allocate_external_computed_blocks( + self, + request_id: str, + num_local_computed_tokens: int, + num_external_computed_tokens: int, + record_for_zeroing: bool = True, + ) -> None: + """Reserve a writable final page; earlier imported pages stay on host.""" + assert num_external_computed_tokens > 0 + num_tokens = num_local_computed_tokens + num_external_computed_tokens + blocks = self.req_to_blocks[request_id] + tail_page = (num_tokens - 1) // self.block_size + if len(blocks) <= tail_page: + blocks.extend([self._null_block] * (tail_page + 1 - len(blocks))) + if blocks[tail_page].is_null: + blocks[tail_page] = self.block_pool.get_new_blocks(1)[0] + + def add_local_computed_blocks( + self, + request_id: str, + new_computed_blocks: Sequence[KVCacheBlock], + num_local_computed_tokens: int, + num_external_computed_tokens: int, + ) -> None: + assert not new_computed_blocks + req_blocks = self.req_to_blocks[request_id] + assert not req_blocks + num_host_pages = cdiv(num_local_computed_tokens, self.block_size) + req_blocks.extend([self._null_block] * num_host_pages) + self.num_cached_block[request_id] = 0 + assert self.coordinator is not None + self.coordinator.commit_computed_blocks(request_id, num_host_pages) + + def allocate_new_blocks( + self, request_id: str, num_tokens: int, num_tokens_main_model: int + ) -> list[KVCacheBlock]: + del num_tokens_main_model + req_blocks = self.req_to_blocks[request_id] + num_new_blocks = cdiv(num_tokens, self.block_size) - len(req_blocks) + if num_new_blocks <= 0: + return [] + new_blocks = self.block_pool.get_new_blocks(num_new_blocks) + req_blocks.extend(new_blocks) + return new_blocks + + def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: + assert self.coordinator is not None + self.coordinator.free(request_id) + return super().pop_blocks_for_free(request_id) + + def adopt_resident_page( + self, request_id: str, block_idx: int, block: KVCacheBlock + ) -> bool: + """Point a null prefix page at a pinned GPU copy of its contents.""" + blocks = self.req_to_blocks.get(request_id) + if blocks is None or block_idx >= len(blocks) or not blocks[block_idx].is_null: + return False + blocks[block_idx] = block + return True + + def get_resident_page(self, request_id: str, block_idx: int) -> KVCacheBlock | None: + blocks = self.req_to_blocks.get(request_id) + if blocks is None or block_idx >= len(blocks): + return None + block = blocks[block_idx] + return None if block.is_null else block + + +def get_manager_for_kv_cache_spec( + kv_cache_spec: KVCacheSpec, + max_in_flight_tokens: int, + max_model_len: int, + role: str | None = None, + **kwargs, +) -> SingleTypeKVCacheManager: + """Get the appropriate manager for a given KVCacheSpec. + + Uses the KVCacheSpecRegistry to look up the manager class, supporting + both built-in and custom specs registered via @register_kv_cache_spec + and KVCacheSpecRegistry.register. + + Args: + kv_cache_spec: The KVCacheSpec instance + max_in_flight_tokens: The max tokens scheduled but not yet settled + (one batch per concurrent step); see `VllmConfig.max_in_flight_tokens` + max_model_len: The maximum context length the model could serve + role: Optional KV cache group role used to select a role-specific + manager + Returns: + An instance of the appropriate SingleTypeKVCacheManager subclass + + """ + manager_class = KVCacheSpecRegistry.get_manager_class(kv_cache_spec, role) + assert manager_class is not None, ( + f"No manager registered for KVCacheSpec {type(kv_cache_spec)}" + ) + # SlidingWindow / ChunkedLocalAttention managers recycle blocks; + # the runtime admission cap must match the recycling-aware bound the + # startup pool sizer uses (single source of truth: the spec method). + # R-SWA also recycles gap blocks but peak physical KV still fits the + # full-attention bound (prefix + window <= max_model_len), so it inherits + # FullAttentionSpec sizing without a separate admission cap. + if isinstance( + kv_cache_spec, + (SlidingWindowSpec, ChunkedLocalAttentionSpec, HiSparseResidentSpec), + ): + kwargs["max_admission_blocks_per_request"] = ( + kv_cache_spec.max_admission_blocks_per_request( + max_in_flight_tokens=max_in_flight_tokens, + max_model_len=max_model_len, + ) + ) + manager = manager_class(kv_cache_spec, **kwargs) + return manager + + +def register_all_kvcache_specs(vllm_config): + """Built-in spec registration""" + KVCacheSpecRegistry.register_role_manager( + KVCacheGroupRole.HISPARSE_SOURCE, HiSparseSourceManager + ) + KVCacheSpecRegistry.register( + FullAttentionSpec, + FullAttentionManager, + uniform_type_base_spec=FullAttentionSpec, + ) + KVCacheSpecRegistry.register( + HiSparseHotSpec, + HiSparseHotManager, + uniform_type_base_spec=HiSparseHotSpec, + ) + KVCacheSpecRegistry.register( + HiSparseResidentSpec, + HiSparseResidentManager, + uniform_type_base_spec=HiSparseResidentSpec, + ) + + KVCacheSpecRegistry.register( + SlidingWindowSpec, + SlidingWindowManager, + uniform_type_base_spec=SlidingWindowSpec, + ) + KVCacheSpecRegistry.register( + CircularBufferSpec, + CircularBufferManager, + uniform_type_base_spec=CircularBufferSpec, + ) + KVCacheSpecRegistry.register( + SlidingWindowMLASpec, + SlidingWindowManager, + uniform_type_base_spec=SlidingWindowMLASpec, + ) + KVCacheSpecRegistry.register( + KpoolTailSpec, + KpoolTailManager, + uniform_type_base_spec=KpoolTailSpec, + ) + + KVCacheSpecRegistry.register( + MambaSpec, MambaManager, uniform_type_base_spec=MambaSpec + ) + KVCacheSpecRegistry.register( + ChunkedLocalAttentionSpec, + ChunkedLocalAttentionManager, + uniform_type_base_spec=ChunkedLocalAttentionSpec, + ) + KVCacheSpecRegistry.register( + CrossAttentionSpec, + CrossAttentionManager, + uniform_type_base_spec=CrossAttentionSpec, + ) + + # FullAttentionSpec subclasses — grouped with FullAttentionSpec + KVCacheSpecRegistry.register( + MLAAttentionSpec, FullAttentionManager, uniform_type_base_spec=FullAttentionSpec + ) + KVCacheSpecRegistry.register( + RSWASpec, RSWAManager, uniform_type_base_spec=FullAttentionSpec + ) + # NOTE(Mengqing): HiddenStateCacheSpec won't take part in + # grouping, thus the uniform_type_base_spec is just a + # placeholder. + KVCacheSpecRegistry.register( + HiddenStateCacheSpec, + FullAttentionManager, + uniform_type_base_spec=FullAttentionSpec, + ) + KVCacheSpecRegistry.register( + SinkFullAttentionSpec, + SinkFullAttentionManager, + uniform_type_base_spec=FullAttentionSpec, + ) + + from vllm.platforms import current_platform + + current_platform.register_custom_kv_cache_specs(vllm_config) diff --git a/vllm/v1/kv_cache_interface.py b/vllm/v1/kv_cache_interface.py index 846e6bcad0..6b113422cd 100644 --- a/vllm/v1/kv_cache_interface.py +++ b/vllm/v1/kv_cache_interface.py @@ -1431,6 +1431,11 @@ class KVCacheTensor: block_stride: int offset: int = 0 # byte offset of layers[0]'s block 0 host_resident: bool = False + # v2.5 side-cache layout: a slot-packed ring tensor views every pool block + # as ``slots_per_block`` equal slots (``block_stride`` is then one slot), so + # its block ids are ``pool_block * slots_per_block + slot``. 1 = one view + # block per pool block, the ordinary case. + slots_per_block: int = 1 class KVCacheGroupRole(str, Enum): @@ -1458,6 +1463,10 @@ class KVCacheGroupSpec: # source group after a connector load. enable_kv_transfer: bool = True role: KVCacheGroupRole = KVCacheGroupRole.DEFAULT + # v2.5 side-cache layout: > 1 when this one-block-per-request ring group + # shares each pool block between that many requests (see + # ``CircularBufferManager.enable_slot_packing``); 1 = one pool block each. + slots_per_block: int = 1 @dataclass @@ -1545,6 +1554,18 @@ class KVCacheConfig: ) return tuple(block_ids[group_id] for group_id in self.transfer_group_ids) + @property + def slot_packed_group_ids(self) -> tuple[int, ...]: + """Groups whose blocks are slots of shared pool blocks (v2.5 side-cache + layout). Their views are not one block per pool block, so whole-block + zeroing must skip them; their bytes lie inside pool blocks the ring + manager owns and that no other group writes.""" + return tuple( + group_id + for group_id, group in enumerate(self.kv_cache_groups) + if group.slots_per_block > 1 + ) + @property def host_group_ids(self) -> tuple[int, ...]: return tuple( @@ -1556,7 +1577,7 @@ class KVCacheConfig: def num_blocks_of(self, tensor: KVCacheTensor) -> int: """Number of blocks addressable by the pool backing ``tensor``.""" if not tensor.host_resident: - return self.num_blocks + return self.num_blocks * tensor.slots_per_block assert self.hisparse_host_num_blocks is not None return self.hisparse_host_num_blocks diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index 140651316e..eb1ac4f9e0 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -866,6 +866,7 @@ class GPUModelRunner(LoRAModelRunnerMixin): kernel_block_sizes=self.kernel_block_sizes, static_forward_context=self.compilation_config.static_forward_context, num_blocks=self.kv_cache_config.num_blocks, + skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids, ) @torch.inference_mode() diff --git a/vllm/v1/worker/gpu_model_runner.py b/vllm/v1/worker/gpu_model_runner.py index 86774b8a60..93a30eb4e9 100644 --- a/vllm/v1/worker/gpu_model_runner.py +++ b/vllm/v1/worker/gpu_model_runner.py @@ -1149,6 +1149,7 @@ class GPUModelRunner( runner_only_attn_layers=self.runner_only_attn_layers, static_forward_context=self.compilation_config.static_forward_context, num_blocks=self.kv_cache_config.num_blocks, + skip_kv_cache_group_ids=self.kv_cache_config.slot_packed_group_ids, ) def _zero_block_ids(self, block_ids: list[int]) -> None: diff --git a/vllm/v1/worker/utils.py b/vllm/v1/worker/utils.py index a107f23bd8..c24b2b0242 100644 --- a/vllm/v1/worker/utils.py +++ b/vllm/v1/worker/utils.py @@ -138,6 +138,7 @@ class KVBlockZeroer: static_forward_context: dict[str, Any], num_blocks: int, runner_only_attn_layers: set[str] | None = None, + skip_kv_cache_group_ids: Iterable[int] = (), ) -> None: """Precompute the absolute-address table for the Triton zeroing kernel. @@ -156,6 +157,13 @@ class KVBlockZeroer: physical block stride and zeroed page span remain independent. Only AttentionSpec layers are processed; Mamba layers are skipped. + + ``skip_kv_cache_group_ids`` (v2.5 side-cache layout): slot-packed ring + groups, whose views hold several slots per pool block. Their claims + never enter whole-block zeroing (the ring-tag path clears their tag words instead), + and with one view block per pool block their pages sat inside the + first attention page, so skipping them leaves the zeroed bytes of + every block unchanged. """ self.device = device # Bounds-guard instrumentation: pool bound for the guard in _zero_kv_blocks_kernel. @@ -174,12 +182,15 @@ class KVBlockZeroer: seg_block_strides: list[int] = [] seg_page_sizes: list[int] = [] + skip_group_ids = set(skip_kv_cache_group_ids) for group in attn_groups_iter: spec = group.kv_cache_spec if not isinstance(spec, AttentionSpec): continue if group.kv_cache_group_id >= len(kernel_block_sizes): continue + if group.kv_cache_group_id in skip_group_ids: + continue kernel_bs = kernel_block_sizes[group.kv_cache_group_id] assert spec.block_size % kernel_bs == 0 for layer_name in group.layer_names: