[Scheduler] Keep the running request's Mamba prefix checkpoint referenced until it finishes
In Mamba align mode a state block leaves the block table once the request has computed one block past it. When that block is hashed it is the request's prefix checkpoint, and releasing it let a long concurrent session evict it, so the next turn re-prefilled the whole conversation. The manager now keeps the newest hashed checkpoint referenced until the request finishes, and admission reserves that block. VLLM_E2_MAMBA_RETAIN_CHECKPOINT (default 1) switches it off. Scheduler tests cover eviction, admission, preemption and freeing. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+22
-3
@@ -450,6 +450,14 @@ if TYPE_CHECKING:
|
||||
# AOT: no. CAPTURE: no -- strictly host-side. off|warn|raise (1 = warn).
|
||||
# the mamba slot-ownership check, v1/worker/mamba_utils.py:74.
|
||||
VLLM_E2_MAMBA_SLOT_OWNERSHIP: str = "off"
|
||||
# v2.5: keep each running request's newest hashed align-mode Mamba
|
||||
# state (its prefix-cache checkpoint) referenced until the request ends,
|
||||
# instead of returning it to the free queue the moment decode passes it
|
||||
# (single_type_kv_cache_manager.MambaManager._retire_state_blocks). Strict
|
||||
# "0"/"1", default ON; "0" restores the old release-on-retire behaviour
|
||||
# for A/B. AOT: no. CAPTURE: no -- scheduler-side block bookkeeping only;
|
||||
# the worker sees the same block tables either way.
|
||||
VLLM_E2_MAMBA_RETAIN_CHECKPOINT: bool = True
|
||||
# Opt-out for the fork startup guards in vllm/v1/e2_guards.py, read at
|
||||
# Qwen4ExpForCausalLM construction. AOT: no. CAPTURE: no -- it changes
|
||||
# nothing itself; it only permits a boot whose identity IS different
|
||||
@@ -933,15 +941,17 @@ def _resolve_rust_cli_path() -> str | None:
|
||||
return raw
|
||||
|
||||
|
||||
def _e2_strict_switch(name: str) -> bool:
|
||||
def _e2_strict_switch(name: str, default: bool = False) -> bool:
|
||||
"""Parse a fork on/off switch that accepts exactly ``"0"`` and ``"1"``.
|
||||
|
||||
Unset means off. Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
|
||||
Unset means ``default`` (off unless the knob says otherwise). Anything else -- ``"true"``, ``"yes"``, ``""``, ``" 1"``,
|
||||
``"01"`` -- is a boot error: a switch that selects a different transport
|
||||
must not be satisfiable by a spelling nobody meant.
|
||||
"""
|
||||
raw = os.environ.get(name)
|
||||
if raw is None or raw == "0":
|
||||
if raw is None:
|
||||
return default
|
||||
if raw == "0":
|
||||
return False
|
||||
if raw == "1":
|
||||
return True
|
||||
@@ -2679,6 +2689,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_SKIP_VERSION_SUFFIX": lambda: bool(
|
||||
int(os.getenv("VLLM_SKIP_VERSION_SUFFIX", "0"))
|
||||
),
|
||||
# Mamba prefix-checkpoint retention (v2.5; default ON, "0" turns it off).
|
||||
# Strict "0"/"1"; read when each MambaManager is constructed.
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT": lambda: _e2_strict_switch(
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT", default=True
|
||||
),
|
||||
# Seconds between `topk-exact-counters` lines from each GPU worker
|
||||
# (0, the default, disables the reader entirely). The line reports the
|
||||
# medium-path exact-restart counters; see
|
||||
@@ -2930,6 +2945,10 @@ def compile_factors() -> dict[str, object]:
|
||||
# configuration factor and is hashed conservatively.
|
||||
"VLLM_E2_PLE_FLAG_CHECK",
|
||||
"VLLM_E2_MAMBA_SLOT_OWNERSHIP",
|
||||
# Scheduler-side block retention (v2.5). The scheduler is never
|
||||
# compiled and the worker's tables are identical in both worlds, so an
|
||||
# A/B flip must reuse the same AOT cache.
|
||||
"VLLM_E2_MAMBA_RETAIN_CHECKPOINT",
|
||||
# Logging cadence for an already-compiled kernel's counters: it gates a
|
||||
# host-side timer and a snapshot op, and is read by no compiled code, so
|
||||
# hashing it would invalidate every compile cache for a log interval.
|
||||
|
||||
@@ -233,6 +233,17 @@ class KVCacheCoordinator(ABC):
|
||||
apply_admission_cap=apply_admission_cap,
|
||||
prefill_end=prefill_end,
|
||||
)
|
||||
if apply_admission_cap:
|
||||
# Full-sequence admission: a retained Mamba checkpoint is
|
||||
# held outside the block table for the request's
|
||||
# lifetime, so reserve it here (v2.5). Per-step
|
||||
# allocation needs no term: a held block is simply not in
|
||||
# the free queue that check reads.
|
||||
num_blocks_to_allocate += (
|
||||
manager.get_num_retained_checkpoint_blocks_to_reserve(
|
||||
request_id, num_tokens_main_model
|
||||
)
|
||||
)
|
||||
return num_blocks_to_allocate
|
||||
|
||||
def allocate_new_computed_blocks(
|
||||
@@ -395,8 +406,15 @@ class KVCacheCoordinator(ABC):
|
||||
request_id: The request ID.
|
||||
|
||||
"""
|
||||
# Retained Mamba checkpoints go back last, after every group's table,
|
||||
# so a finished request's checkpoint is never evicted ahead of the
|
||||
# attention prefix it resumes (v2.5).
|
||||
retained: list[KVCacheBlock] = []
|
||||
for manager in self.single_type_managers:
|
||||
retained.extend(manager.pop_retained_checkpoints(request_id))
|
||||
manager.free(request_id)
|
||||
if retained:
|
||||
self.block_pool.free_blocks(retained)
|
||||
|
||||
def pop_blocks_for_free(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""Pop the request's bookkeeping from all single-type managers and
|
||||
@@ -414,12 +432,16 @@ class KVCacheCoordinator(ABC):
|
||||
|
||||
"""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
retained: list[KVCacheBlock] = []
|
||||
for manager in self.single_type_managers:
|
||||
retained.extend(manager.pop_retained_checkpoints(request_id))
|
||||
# Reverse within each group so the concatenation is already in
|
||||
# eviction order, matching `free()`, which frees each group in its
|
||||
# own reversed `free_blocks()` call. Reversing the flat list
|
||||
# instead would emit whole groups in the wrong order.
|
||||
blocks.extend(reversed(manager.pop_blocks_for_free(request_id)))
|
||||
# Retained checkpoints last, as in `free()`.
|
||||
blocks.extend(retained)
|
||||
return blocks
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> list[int]:
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from vllm import envs
|
||||
from vllm.distributed.kv_events import MEDIUM_CPU
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -608,6 +609,22 @@ class SingleTypeKVCacheManager(ABC):
|
||||
# Free blocks in reverse order so that the tail blocks are freed first.
|
||||
self.block_pool.free_blocks(reversed(self.pop_blocks_for_free(request_id)))
|
||||
|
||||
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
|
||||
"""Pop the prefix-cache checkpoints held for ``request_id`` outside its
|
||||
block table, without freeing them. Only align-mode Mamba holds any
|
||||
(see ``MambaManager._retire_state_blocks``). The coordinator frees them
|
||||
after every group's table, so they are the request's last blocks to
|
||||
become eviction candidates."""
|
||||
return []
|
||||
|
||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||
self, request_id: str, num_tokens: int
|
||||
) -> int:
|
||||
"""Blocks the admission gate must add for checkpoints this request
|
||||
will hold outside its block table (0 for every manager but align-mode
|
||||
Mamba with retention on)."""
|
||||
return 0
|
||||
|
||||
@abstractmethod
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""Get the number of common prefix blocks for all requests with allocated
|
||||
@@ -1535,6 +1552,80 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
# connector; a request that finishes first hands off this table
|
||||
# source directly.
|
||||
self._producer_partial_tail_reqs: dict[str, tuple[KVCacheBlock, int]] = {}
|
||||
# v2.5. In align mode a state block leaves the block table (it is
|
||||
# nulled and released) as soon as the request computes one block past
|
||||
# it -- `_remove_blocks_in_range` / `remove_skipped_blocks`. When that
|
||||
# block carries a prefix-cache hash it is the request's GDN checkpoint
|
||||
# (the replay boundary under retention interval 0), and releasing it
|
||||
# mid-request puts it in the free queue AHEAD of the request's own
|
||||
# attention blocks, which stay referenced until finish. Concurrent
|
||||
# prefills then pop it first; since the hybrid hit is the shortest
|
||||
# group's, that single block zeroes the next turn's reuse. With
|
||||
# retention on, the newest hashed retired state stays referenced here
|
||||
# (at most one per request per group) and is released only when the
|
||||
# request is freed, after its attention blocks (coordinator.free).
|
||||
self._retain_checkpoint: bool = (
|
||||
self.mamba_cache_mode == "align"
|
||||
and self.enable_caching
|
||||
and envs.VLLM_E2_MAMBA_RETAIN_CHECKPOINT
|
||||
)
|
||||
# {request_id: (logical block index, block)}
|
||||
self._retained_checkpoints: dict[str, tuple[int, KVCacheBlock]] = {}
|
||||
if self.mamba_cache_mode == "align" and self.enable_caching:
|
||||
# The A/B evidence line: boot logs must show which arm ran.
|
||||
logger.info_once(
|
||||
"Mamba align prefix-checkpoint retention: %s "
|
||||
"(VLLM_E2_MAMBA_RETAIN_CHECKPOINT)",
|
||||
"on" if self._retain_checkpoint else "off",
|
||||
)
|
||||
|
||||
def _retire_state_blocks(
|
||||
self, request_id: str, retired: list[tuple[int, KVCacheBlock]]
|
||||
) -> None:
|
||||
"""Release align-mode state blocks the request no longer reads.
|
||||
|
||||
``retired`` holds (logical index, block) pairs already nulled out of
|
||||
the block table, in the eviction order the caller wants. With
|
||||
retention on, the newest hashed one is kept referenced as the request's
|
||||
checkpoint instead, and the checkpoint it supersedes (if any) is
|
||||
released now -- so the request never holds more than one.
|
||||
"""
|
||||
if self._retain_checkpoint:
|
||||
newest: tuple[int, KVCacheBlock] | None = None
|
||||
for idx, block in retired:
|
||||
if block.block_hash is not None and (
|
||||
newest is None or idx > newest[0]
|
||||
):
|
||||
newest = (idx, block)
|
||||
held = self._retained_checkpoints.get(request_id)
|
||||
if newest is not None and (held is None or newest[0] > held[0]):
|
||||
self._retained_checkpoints[request_id] = newest
|
||||
retired = [entry for entry in retired if entry[1] is not newest[1]]
|
||||
if held is not None:
|
||||
# Superseded: it ages from here like any cached free block.
|
||||
retired.append(held)
|
||||
if retired:
|
||||
self.block_pool.free_blocks([block for _, block in retired])
|
||||
|
||||
def pop_retained_checkpoints(self, request_id: str) -> list[KVCacheBlock]:
|
||||
if self.mamba_cache_mode != "align":
|
||||
return []
|
||||
held = self._retained_checkpoints.pop(request_id, None)
|
||||
return [] if held is None else [held[1]]
|
||||
|
||||
def get_num_retained_checkpoint_blocks_to_reserve(
|
||||
self, request_id: str, num_tokens: int
|
||||
) -> int:
|
||||
# A request that already holds a checkpoint only ever swaps it. One
|
||||
# that does not will hold one once a hashed state retires, which needs
|
||||
# at least one state block behind the running one.
|
||||
if (
|
||||
not self._retain_checkpoint
|
||||
or request_id in self._retained_checkpoints
|
||||
or num_tokens <= self.block_size
|
||||
):
|
||||
return 0
|
||||
return 1
|
||||
|
||||
@classmethod
|
||||
def find_longest_cache_hit(
|
||||
@@ -1694,15 +1785,14 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
last_block = min(last_block, len(blocks))
|
||||
if first_block >= last_block:
|
||||
return
|
||||
freed: list[KVCacheBlock] = []
|
||||
freed: list[tuple[int, KVCacheBlock]] = []
|
||||
# Mamba prefill leaves null gaps between states awaiting retirement.
|
||||
for i in range(last_block - 1, first_block - 1, -1):
|
||||
if blocks[i].is_null:
|
||||
continue
|
||||
freed.append(blocks[i])
|
||||
freed.append((i, blocks[i]))
|
||||
blocks[i] = self._null_block
|
||||
if freed:
|
||||
self.block_pool.free_blocks(freed)
|
||||
self._retire_state_blocks(request_id, freed)
|
||||
self._num_retired_blocks[request_id] = last_block
|
||||
|
||||
def remove_skipped_blocks(
|
||||
@@ -1732,8 +1822,11 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
):
|
||||
blocks = self.req_to_blocks[request_id]
|
||||
if blocks[last_state_block_idx] != self._null_block:
|
||||
self.block_pool.free_blocks([blocks[last_state_block_idx]])
|
||||
block = blocks[last_state_block_idx]
|
||||
blocks[last_state_block_idx] = self._null_block
|
||||
self._retire_state_blocks(
|
||||
request_id, [(last_state_block_idx, block)]
|
||||
)
|
||||
|
||||
def get_num_common_prefix_blocks(self, running_request_id: str) -> int:
|
||||
"""Cascade attention is not supported by mamba"""
|
||||
@@ -2046,6 +2139,12 @@ class MambaManager(SingleTypeKVCacheManager):
|
||||
for entry in self._pending_boundary_state_offloads
|
||||
if entry[0] != request_id
|
||||
]
|
||||
# The coordinator pops the retained checkpoint first and frees it
|
||||
# after every group. A direct caller still gets it back, first in
|
||||
# allocation order, so the caller's reversed free releases it last.
|
||||
retained = self.pop_retained_checkpoints(request_id)
|
||||
if retained:
|
||||
return retained + super().pop_blocks_for_free(request_id)
|
||||
return super().pop_blocks_for_free(request_id)
|
||||
|
||||
def get_num_skipped_tokens(self, num_computed_tokens: int) -> int:
|
||||
|
||||
Reference in New Issue
Block a user