From 010521f10d05b4167dfe4b105276cf78a9fc2bf9 Mon Sep 17 00:00:00 2001 From: MOVZX Date: Fri, 2 Oct 2026 04:09:20 -0400 Subject: [PATCH] [Scheduler] Keep the running request's Mamba prefix checkpoint referenced until it finishes In Mamba align mode a state block leaves the block table once the request has computed one block past it. When that block is hashed it is the request's prefix checkpoint, and releasing it let a long concurrent session evict it, so the next turn re-prefilled the whole conversation. The manager now keeps the newest hashed checkpoint referenced until the request finishes, and admission reserves that block. VLLM_E2_MAMBA_RETAIN_CHECKPOINT (default 1) switches it off. Scheduler tests cover eviction, admission, preemption and freeing. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/envs.py | 25 ++++- vllm/v1/core/kv_cache_coordinator.py | 22 ++++ vllm/v1/core/single_type_kv_cache_manager.py | 109 ++++++++++++++++++- 3 files changed, 148 insertions(+), 8 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 653da51b34..42e72f3652 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -450,6 +450,14 @@ if TYPE_CHECKING: # AOT: no. CAPTURE: no -- strictly host-side. off|warn|raise (1 = warn). # the mamba slot-ownership check, v1/worker/mamba_utils.py:74. VLLM_E2_MAMBA_SLOT_OWNERSHIP: str = "off" + # v2.5: keep each running request's newest hashed align-mode Mamba + # state (its prefix-cache checkpoint) referenced until the request ends, + # instead of returning it to the free queue the moment decode passes it + # (single_type_kv_cache_manager.MambaManager._retire_state_blocks). Strict + # "0"/"1", default ON; "0" restores the old release-on-retire behaviour + # for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only; + # the worker sees the same block tables either way. + VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True # Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at # Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes # nothing itself; it only permits a boot whose identity IS different @@ -933,15 +941,17 @@ def _resolve_rust_cli_path() -> str | None: return raw -def _e2_strict_switch(name: str) -> bool: +def _e2_strict_switch(name: str, default: bool = False) -> bool: """Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``. - Unset means off. Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``, + Unset means ``default`` (off unless the knob says otherwise). Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``, ``"01"`` -- is a boot error: a switch that selects a different transport must not be satisfiable by a spelling nobody meant. """ raw = os.environ.get(name) - if raw is None or raw == "0": + if raw is None: + return default + if raw == "0": return False if raw == "1": return True @@ -2679,6 +2689,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_SKIP_VERSION_SUFFIX": lambda: bool( int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0")) ), + # Mamba prefix-checkpoint retention (v2.5; default ON, "0" turns it off). + # Strict "0"/"1"; read when each MambaManager is constructed. + "VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch( + "VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True + ), # Seconds between `topk-exact-counters` lines from each GPU worker # (0, the default, disables the reader entirely). The line reports the # medium-path exact-restart counters; see @@ -2930,6 +2945,10 @@ def compile_factors() -> dict[str, object]: # configuration factor and is hashed conservatively. "VLLM_E2_PLE_FLAG_CHECK", "VLLM_E2_MAMBA_SLOT_OWNERSHIP", + # Scheduler-side block retention (v2.5). The scheduler is never + # compiled and the worker's tables are identical in both worlds, so an + # A/B flip must reuse the same AOT cache. + "VLLM_E2_MAMBA_RETAIN_CHECKPOINT", # Logging cadence for an already-compiled kernel's counters: it gates a # host-side timer and a snapshot op, and is read by no compiled code, so # hashing it would invalidate every compile cache for a log interval. diff --git a/vllm/v1/core/kv_cache_coordinator.py b/vllm/v1/core/kv_cache_coordinator.py index f457bb9fdd..1a4c65e38c 100644 --- a/vllm/v1/core/kv_cache_coordinator.py +++ b/vllm/v1/core/kv_cache_coordinator.py @@ -233,6 +233,17 @@ class KVCacheCoordinator(ABC): apply_admission_cap=apply_admission_cap, prefill_end=prefill_end, ) + if apply_admission_cap: + # Full-sequence admission: a retained Mamba checkpoint is + # held outside the block table for the request's + # lifetime, so reserve it here (v2.5). Per-step + # allocation needs no term: a held block is simply not in + # the free queue that check reads. + num_blocks_to_allocate += ( + manager.get_num_retained_checkpoint_blocks_to_reserve( + request_id, num_tokens_main_model + ) + ) return num_blocks_to_allocate def allocate_new_computed_blocks( @@ -395,8 +406,15 @@ class KVCacheCoordinator(ABC): request_id: The request ID. """ + # Retained Mamba checkpoints go back last, after every group's table, + # so a finished request's checkpoint is never evicted ahead of the + # attention prefix it resumes (v2.5). + retained: list[KVCacheBlock] = [] for manager in self.single_type_managers: + retained.extend(manager.pop_retained_checkpoints(request_id)) manager.free(request_id) + if retained: + self.block_pool.free_blocks(retained) def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]: """Pop the request's bookkeeping from all single-type managers and @@ -414,12 +432,16 @@ class KVCacheCoordinator(ABC): """ blocks: list[KVCacheBlock] = [] + retained: list[KVCacheBlock] = [] for manager in self.single_type_managers: + retained.extend(manager.pop_retained_checkpoints(request_id)) # Reverse within each group so the concatenation is already in # eviction order, matching `free()`, which frees each group in its # own reversed `free_blocks()` call. Reversing the flat list # instead would emit whole groups in the wrong order. blocks.extend(reversed(manager.pop_blocks_for_free(request_id))) + # Retained checkpoints last, as in `free()`. + blocks.extend(retained) return blocks def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]: diff --git a/vllm/v1/core/single_type_kv_cache_manager.py b/vllm/v1/core/single_type_kv_cache_manager.py index e4bdbc62e7..20d9bcc34f 100644 --- a/vllm/v1/core/single_type_kv_cache_manager.py +++ b/vllm/v1/core/single_type_kv_cache_manager.py @@ -6,6 +6,7 @@ from collections import defaultdict from collections.abc import Sequence from typing import TYPE_CHECKING, ClassVar +from vllm import envs from vllm.distributed.kv_events import MEDIUM_CPU from vllm.logger import init_logger from vllm.utils.math_utils import cdiv @@ -608,6 +609,22 @@ class SingleTypeKVCacheManager(ABC): # Free blocks in reverse order so that the tail blocks are freed first. self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id))) + def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]: + """Pop the prefix-cache checkpoints held for ``request_id`` outside its + block table, without freeing them. Only align-mode Mamba holds any + (see ``MambaManager._retire_state_blocks``). The coordinator frees them + after every group's table, so they are the request's last blocks to + become eviction candidates.""" + return [] + + def get_num_retained_checkpoint_blocks_to_reserve( + self, request_id: str, num_tokens: int + ) -> int: + """Blocks the admission gate must add for checkpoints this request + will hold outside its block table (0 for every manager but align-mode + Mamba with retention on).""" + return 0 + @abstractmethod def get_num_common_prefix_blocks(self, running_request_id: str) -> int: """Get the number of common prefix blocks for all requests with allocated @@ -1535,6 +1552,80 @@ class MambaManager(SingleTypeKVCacheManager): # connector; a request that finishes first hands off this table # source directly. self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {} + # v2.5. In align mode a state block leaves the block table (it is + # nulled and released) as soon as the request computes one block past + # it -- `_remove_blocks_in_range` / `remove_skipped_blocks`. When that + # block carries a prefix-cache hash it is the request's GDN checkpoint + # (the replay boundary under retention interval 0), and releasing it + # mid-request puts it in the free queue AHEAD of the request's own + # attention blocks, which stay referenced until finish. Concurrent + # prefills then pop it first; since the hybrid hit is the shortest + # group's, that single block zeroes the next turn's reuse. With + # retention on, the newest hashed retired state stays referenced here + # (at most one per request per group) and is released only when the + # request is freed, after its attention blocks (coordinator.free). + self._retain_checkpoint: bool = ( + self.mamba_cache_mode == "align" + and self.enable_caching + and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT + ) + # {request_id: (logical block index, block)} + self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {} + if self.mamba_cache_mode == "align" and self.enable_caching: + # The A/B evidence line: boot logs must show which arm ran. + logger.info_once( + "Mamba align prefix-checkpoint retention: %s " + "(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)", + "on" if self._retain_checkpoint else "off", + ) + + def _retire_state_blocks( + self, request_id: str, retired: list[tuple[int, KVCacheBlock]] + ) -> None: + """Release align-mode state blocks the request no longer reads. + + ``retired`` holds (logical index, block) pairs already nulled out of + the block table, in the eviction order the caller wants. With + retention on, the newest hashed one is kept referenced as the request's + checkpoint instead, and the checkpoint it supersedes (if any) is + released now -- so the request never holds more than one. + """ + if self._retain_checkpoint: + newest: tuple[int, KVCacheBlock] | None = None + for idx, block in retired: + if block.block_hash is not None and ( + newest is None or idx > newest[0] + ): + newest = (idx, block) + held = self._retained_checkpoints.get(request_id) + if newest is not None and (held is None or newest[0] > held[0]): + self._retained_checkpoints[request_id] = newest + retired = [entry for entry in retired if entry[1] is not newest[1]] + if held is not None: + # Superseded: it ages from here like any cached free block. + retired.append(held) + if retired: + self.block_pool.free_blocks([block for _, block in retired]) + + def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]: + if self.mamba_cache_mode != "align": + return [] + held = self._retained_checkpoints.pop(request_id, None) + return [] if held is None else [held[1]] + + def get_num_retained_checkpoint_blocks_to_reserve( + self, request_id: str, num_tokens: int + ) -> int: + # A request that already holds a checkpoint only ever swaps it. One + # that does not will hold one once a hashed state retires, which needs + # at least one state block behind the running one. + if ( + not self._retain_checkpoint + or request_id in self._retained_checkpoints + or num_tokens <= self.block_size + ): + return 0 + return 1 @classmethod def find_longest_cache_hit( @@ -1694,15 +1785,14 @@ class MambaManager(SingleTypeKVCacheManager): last_block = min(last_block, len(blocks)) if first_block >= last_block: return - freed: list[KVCacheBlock] = [] + freed: list[tuple[int, KVCacheBlock]] = [] # Mamba prefill leaves null gaps between states awaiting retirement. for i in range(last_block - 1, first_block - 1, -1): if blocks[i].is_null: continue - freed.append(blocks[i]) + freed.append((i, blocks[i])) blocks[i] = self._null_block - if freed: - self.block_pool.free_blocks(freed) + self._retire_state_blocks(request_id, freed) self._num_retired_blocks[request_id] = last_block def remove_skipped_blocks( @@ -1732,8 +1822,11 @@ class MambaManager(SingleTypeKVCacheManager): ): blocks = self.req_to_blocks[request_id] if blocks[last_state_block_idx] != self._null_block: - self.block_pool.free_blocks([blocks[last_state_block_idx]]) + block = blocks[last_state_block_idx] blocks[last_state_block_idx] = self._null_block + self._retire_state_blocks( + request_id, [(last_state_block_idx, block)] + ) def get_num_common_prefix_blocks(self, running_request_id: str) -> int: """Cascade attention is not supported by mamba""" @@ -2046,6 +2139,12 @@ class MambaManager(SingleTypeKVCacheManager): for entry in self._pending_boundary_state_offloads if entry[0] != request_id ] + # The coordinator pops the retained checkpoint first and frees it + # after every group. A direct caller still gets it back, first in + # allocation order, so the caller's reversed free releases it last. + retained = self.pop_retained_checkpoints(request_id) + if retained: + return retained + super().pop_blocks_for_free(request_id) return super().pop_blocks_for_free(request_id) def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: