[Scheduler] Keep the running request's Mamba prefix checkpoint referenced until it finishes

In Mamba align mode a state block leaves the block table once the request has computed one block past it. When
that block is hashed it is the request's prefix checkpoint, and releasing it let a long concurrent session evict
it, so the next turn re-prefilled the whole conversation. The manager now keeps the newest hashed checkpoint
referenced until the request finishes, and admission reserves that block. VLLM_E2_MAMBA_RETAIN_CHECKPOINT
(default 1) switches it off. Scheduler tests cover eviction, admission, preemption and freeing.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:13:06 +07:00
co-authored by Hinata
parent 0f72060d66
commit 010521f10d
3 changed files with 148 additions and 8 deletions
+22 -3
View File
@@ -450,6 +450,14 @@ if TYPE_CHECKING:
# AOT: no. CAPTURE: no -- strictly host-side. off|warn|raise (1 = warn).
# the mamba slot-ownership check, v1/worker/mamba_utils.py:74.
VLLM_E2_MAMBA_SLOT_OWNERSHIP: str = "off"
# v2.5: keep each running request's newest hashed align-mode Mamba
# state (its prefix-cache checkpoint) referenced until the request ends,
# instead of returning it to the free queue the moment decode passes it
# (single_type_kv_cache_manager.MambaManager._retire_state_blocks). Strict
# "0"/"1", default ON; "0" restores the old release-on-retire behaviour
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
# the worker sees the same block tables either way.
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
# nothing itself; it only permits a boot whose identity IS different
@@ -933,15 +941,17 @@ def _resolve_rust_cli_path() -> str | None:
return raw
def _e2_strict_switch(name: str) -> bool:
def _e2_strict_switch(name: str, default: bool = False) -> bool:
"""Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``.
Unset means off. Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
Unset means ``default`` (off unless the knob says otherwise). Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
``"01"`` -- is a boot error: a switch that selects a different transport
must not be satisfiable by a spelling nobody meant.
"""
raw = os.environ.get(name)
if raw is None or raw == "0":
if raw is None:
return default
if raw == "0":
return False
if raw == "1":
return True
@@ -2679,6 +2689,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
),
# Mamba prefix-checkpoint retention (v2.5; default ON, "0" turns it off).
# Strict "0"/"1"; read when each MambaManager is constructed.
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
),
# Seconds between `topk-exact-counters` lines from each GPU worker
# (0, the default, disables the reader entirely). The line reports the
# medium-path exact-restart counters; see
@@ -2930,6 +2945,10 @@ def compile_factors() -> dict[str, object]:
# configuration factor and is hashed conservatively.
"VLLM_E2_PLE_FLAG_CHECK",
"VLLM_E2_MAMBA_SLOT_OWNERSHIP",
# Scheduler-side block retention (v2.5). The scheduler is never
# compiled and the worker's tables are identical in both worlds, so an
# A/B flip must reuse the same AOT cache.
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT",
# Logging cadence for an already-compiled kernel's counters: it gates a
# host-side timer and a snapshot op, and is read by no compiled code, so
# hashing it would invalidate every compile cache for a log interval.
+22
View File
@@ -233,6 +233,17 @@ class KVCacheCoordinator(ABC):
apply_admission_cap=apply_admission_cap,
prefill_end=prefill_end,
)
if apply_admission_cap:
# Full-sequence admission: a retained Mamba checkpoint is
# held outside the block table for the request's
# lifetime, so reserve it here (v2.5). Per-step
# allocation needs no term: a held block is simply not in
# the free queue that check reads.
num_blocks_to_allocate += (
manager.get_num_retained_checkpoint_blocks_to_reserve(
request_id, num_tokens_main_model
)
)
return num_blocks_to_allocate
def allocate_new_computed_blocks(
@@ -395,8 +406,15 @@ class KVCacheCoordinator(ABC):
request_id: The request ID.
"""
# Retained Mamba checkpoints go back last, after every group's table,
# so a finished request's checkpoint is never evicted ahead of the
# attention prefix it resumes (v2.5).
retained: list[KVCacheBlock] = []
for manager in self.single_type_managers:
retained.extend(manager.pop_retained_checkpoints(request_id))
manager.free(request_id)
if retained:
self.block_pool.free_blocks(retained)
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
"""Pop the request's bookkeeping from all single-type managers and
@@ -414,12 +432,16 @@ class KVCacheCoordinator(ABC):
"""
blocks: list[KVCacheBlock] = []
retained: list[KVCacheBlock] = []
for manager in self.single_type_managers:
retained.extend(manager.pop_retained_checkpoints(request_id))
# Reverse within each group so the concatenation is already in
# eviction order, matching `free()`, which frees each group in its
# own reversed `free_blocks()` call. Reversing the flat list
# instead would emit whole groups in the wrong order.
blocks.extend(reversed(manager.pop_blocks_for_free(request_id)))
# Retained checkpoints last, as in `free()`.
blocks.extend(retained)
return blocks
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
+104 -5
View File
@@ -6,6 +6,7 @@ from collections import defaultdict
from collections.abc import Sequence
from typing import TYPE_CHECKING, ClassVar
from vllm import envs
from vllm.distributed.kv_events import MEDIUM_CPU
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
@@ -608,6 +609,22 @@ class SingleTypeKVCacheManager(ABC):
# Free blocks in reverse order so that the tail blocks are freed first.
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
"""Pop the prefix-cache checkpoints held for ``request_id`` outside its
block table, without freeing them. Only align-mode Mamba holds any
(see ``MambaManager._retire_state_blocks``). The coordinator frees them
after every group's table, so they are the request's last blocks to
become eviction candidates."""
return []
def get_num_retained_checkpoint_blocks_to_reserve(
self, request_id: str, num_tokens: int
) -> int:
"""Blocks the admission gate must add for checkpoints this request
will hold outside its block table (0 for every manager but align-mode
Mamba with retention on)."""
return 0
@abstractmethod
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""Get the number of common prefix blocks for all requests with allocated
@@ -1535,6 +1552,80 @@ class MambaManager(SingleTypeKVCacheManager):
# connector; a request that finishes first hands off this table
# source directly.
self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {}
# v2.5. In align mode a state block leaves the block table (it is
# nulled and released) as soon as the request computes one block past
# it -- `_remove_blocks_in_range` / `remove_skipped_blocks`. When that
# block carries a prefix-cache hash it is the request's GDN checkpoint
# (the replay boundary under retention interval 0), and releasing it
# mid-request puts it in the free queue AHEAD of the request's own
# attention blocks, which stay referenced until finish. Concurrent
# prefills then pop it first; since the hybrid hit is the shortest
# group's, that single block zeroes the next turn's reuse. With
# retention on, the newest hashed retired state stays referenced here
# (at most one per request per group) and is released only when the
# request is freed, after its attention blocks (coordinator.free).
self._retain_checkpoint: bool = (
self.mamba_cache_mode == "align"
and self.enable_caching
and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT
)
# {request_id: (logical block index, block)}
self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {}
if self.mamba_cache_mode == "align" and self.enable_caching:
# The A/B evidence line: boot logs must show which arm ran.
logger.info_once(
"Mamba align prefix-checkpoint retention: %s "
"(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)",
"on" if self._retain_checkpoint else "off",
)
def _retire_state_blocks(
self, request_id: str, retired: list[tuple[int, KVCacheBlock]]
) -> None:
"""Release align-mode state blocks the request no longer reads.
``retired`` holds (logical index, block) pairs already nulled out of
the block table, in the eviction order the caller wants. With
retention on, the newest hashed one is kept referenced as the request's
checkpoint instead, and the checkpoint it supersedes (if any) is
released now -- so the request never holds more than one.
"""
if self._retain_checkpoint:
newest: tuple[int, KVCacheBlock] | None = None
for idx, block in retired:
if block.block_hash is not None and (
newest is None or idx > newest[0]
):
newest = (idx, block)
held = self._retained_checkpoints.get(request_id)
if newest is not None and (held is None or newest[0] > held[0]):
self._retained_checkpoints[request_id] = newest
retired = [entry for entry in retired if entry[1] is not newest[1]]
if held is not None:
# Superseded: it ages from here like any cached free block.
retired.append(held)
if retired:
self.block_pool.free_blocks([block for _, block in retired])
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
if self.mamba_cache_mode != "align":
return []
held = self._retained_checkpoints.pop(request_id, None)
return [] if held is None else [held[1]]
def get_num_retained_checkpoint_blocks_to_reserve(
self, request_id: str, num_tokens: int
) -> int:
# A request that already holds a checkpoint only ever swaps it. One
# that does not will hold one once a hashed state retires, which needs
# at least one state block behind the running one.
if (
not self._retain_checkpoint
or request_id in self._retained_checkpoints
or num_tokens <= self.block_size
):
return 0
return 1
@classmethod
def find_longest_cache_hit(
@@ -1694,15 +1785,14 @@ class MambaManager(SingleTypeKVCacheManager):
last_block = min(last_block, len(blocks))
if first_block >= last_block:
return
freed: list[KVCacheBlock] = []
freed: list[tuple[int, KVCacheBlock]] = []
# Mamba prefill leaves null gaps between states awaiting retirement.
for i in range(last_block - 1, first_block - 1, -1):
if blocks[i].is_null:
continue
freed.append(blocks[i])
freed.append((i, blocks[i]))
blocks[i] = self._null_block
if freed:
self.block_pool.free_blocks(freed)
self._retire_state_blocks(request_id, freed)
self._num_retired_blocks[request_id] = last_block
def remove_skipped_blocks(
@@ -1732,8 +1822,11 @@ class MambaManager(SingleTypeKVCacheManager):
):
blocks = self.req_to_blocks[request_id]
if blocks[last_state_block_idx] != self._null_block:
self.block_pool.free_blocks([blocks[last_state_block_idx]])
block = blocks[last_state_block_idx]
blocks[last_state_block_idx] = self._null_block
self._retire_state_blocks(
request_id, [(last_state_block_idx, block)]
)
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
"""Cascade attention is not supported by mamba"""
@@ -2046,6 +2139,12 @@ class MambaManager(SingleTypeKVCacheManager):
for entry in self._pending_boundary_state_offloads
if entry[0] != request_id
]
# The coordinator pops the retained checkpoint first and frees it
# after every group. A direct caller still gets it back, first in
# allocation order, so the caller's reversed free releases it last.
retained = self.pop_retained_checkpoints(request_id)
if retained:
return retained + super().pop_blocks_for_free(request_id)
return super().pop_blocks_for_free(request_id)
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int: