From 633c758d4c9b1f1a80575ab0767e10682533641d Mon Sep 17 00:00:00 2001 From: MOVZX Date: Tue, 6 Oct 2026 02:49:52 -0400 Subject: [PATCH] [Scheduler] Pin a waiting request's prefix-cache hit until it is admitted A follow-up turn that waits for admission holds no reference on its cached prefix. With a prefix unit finer than the block, its whole hit depends on every one of those blocks: the resume key sits in the top attention block, and the only GDN state is the held tail. So the first of them that another request's allocation popped from the free queue turned the next turn into a cold prefill; that was usually its own top block, which is freed first. Under a long-context multi-turn agent load (8 conversations, 80-100K-token prompts) this gave 4-5 total misses per 300 s at unit 64/512 and none at unit 4096. All of them happened while the pool was at 88-99 % with one or two conversations waiting, and the conversation that missed was usually the next victim too. Now the manager can pin a waiter's hit: - pin_waiting_hit looks the request up and takes a ref on every non-null hit block in every group. It also pins the block holding the EAGLE/MTP proof unit: one unit past the key, dropped from the returned hit, and a different block when the key ends a block. The pinned blocks stay out of the free queue. - release_pinned_hit drops a pin; yield_pinned_hits drops the most recently pinned waiters' first. - allocate_slots releases the request's pin when it succeeds (its table holds the hit from then on), free releases it (abort, finish), and reset_prefix_cache yields every pin before resetting. Pinned blocks are referenced, so admission and the watermark see them as used. The scheduler pins the front of the waiting queue after each scheduling pass: arrival order, at most MAX_PINNED_WAITERS = 8, only requests that have not computed anything yet, never with a KV connector. It does this also while the running set is full, when a waiter is not even looked up. A pin never costs anyone blocks. A running request that fails to allocate yields one waiter's pin and retries before any preemption victim is chosen. A head-of-line waiter whose admission fails yields the pins of the waiters behind it and retries once. When pins are active the scheduler logs one startup line saying so, with the bound, the hash unit and the block size. On a closed-loop CPU model of that load (6-8 conversations, 1-2 waiting, pool 88-100 %), total misses went from 3/1/1/5 to 2/0/0/0 with every waiter pinned. The remaining two sit at 99.2 % use, where the running set needed the pinned blocks themselves (two yields, 65 blocks). Unit 4096 is unaffected except where a waiter was losing its boundary block. Tests: - A pinned waiter survives another follow-up's admission and a burst, and resumes at its tail (a total miss without the pin). - Pool full and a running request needs a block: the pin yields, nobody is preempted, and the waiter is still admitted later. - Pins are released on abort, admission, yield and reset without leaks; the bound is respected; yield order is newest first. - The scheduler's pass pins in arrival order up to the bound and skips partly computed requests and connector setups. Co-Authored-By: Hinata Signed-off-by: MOVZX --- vllm/v1/core/kv_cache_manager.py | 112 +++++++++++++++++++++++++++++++ vllm/v1/core/sched/scheduler.py | 70 +++++++++++++++++++ 2 files changed, 182 insertions(+) diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 4e358da2ba..c31af014bf 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -223,6 +223,21 @@ class KVCacheManager: tuple(() for _ in range(self.num_kv_cache_groups)) ) + # Prefix-cache hits pinned for requests waiting for admission + # ({request_id: blocks}). A waiting follow-up turn holds no ref on its + # cached prefix, and with a prefix unit finer than the block its + # whole hit depends on every one of those blocks (the resume key sits + # in the top block); under pool pressure other requests' allocations + # popped them from the free queue and the turn re-prefilled from 0 + # (under a long-context multi-turn agent load: 4-5 total misses per + # 300 s at unit 64/512, none at 4096). Pinned blocks stay out of the + # free queue, so they count against admission and the watermark like + # any referenced block; they + # are released on admission (the table takes over), on free, and + # whenever a running request would otherwise fail to allocate + # (``yield_pinned_hits``), so a pin never causes a preemption. + self._pinned_hits: dict[str, list[KVCacheBlock]] = {} + @property def usage(self) -> float: """Get the KV cache usage. @@ -602,6 +617,7 @@ class KVCacheManager: # P/D: delay caching blocks if we have to recv from # remote. Update state for locally cached blocks. if delay_cache_blocks: + self.release_pinned_hit(request.request_id) return self.create_kv_cache_blocks(new_blocks) # NOTE(woosuk): We want to commit (cache) up to num_local_computed_tokens @@ -615,6 +631,8 @@ class KVCacheManager: ) self.coordinator.cache_blocks(request, num_tokens_to_cache) + # Admitted: the table now references the hit, the pin is redundant. + self.release_pinned_hit(request.request_id) return self.create_kv_cache_blocks(new_blocks) def free(self, request: Request) -> None: @@ -626,8 +644,100 @@ class KVCacheManager: request: The request to free the blocks. """ + self.release_pinned_hit(request.request_id) self.coordinator.free(request.request_id) + # Waiting requests whose prefix-cache hit is pinned at once (arrival + # order). Pins never block anyone (they yield before a running request + # fails to allocate), so the bound only caps the lookup work per step. + MAX_PINNED_WAITERS = 8 + + @property + def num_pinned_waiters(self) -> int: + return len(self._pinned_hits) + + @property + def num_pinned_blocks(self) -> int: + return sum(len(blocks) for blocks in self._pinned_hits.values()) + + def pin_waiting_hit(self, request: Request) -> int: + """Pin the request's current prefix-cache hit while it waits for + admission: every non-null hit block in every group (the attention + prefix up to the key and the GDN state it resumes from) takes a ref + and leaves the free queue. Idempotent. Returns the number of blocks + pinned now (0 if nothing hit, already pinned, caching is off, or the + waiter bound is reached).""" + request_id = request.request_id + if ( + request_id in self._pinned_hits + or not self.enable_caching + or not self.prefix_cache_lookup_enabled(request) + or len(self._pinned_hits) >= self.MAX_PINNED_WAITERS + ): + return 0 + computed, num_hit, _ = self.get_computed_blocks(request) + blocks = [ + block + for group in computed.blocks + for block in group + if not block.is_null + ] + if num_hit == 0 or not blocks: + return 0 + # The EAGLE/MTP proof: fine-grained attention hits match one hash + # unit past the key and drop it, so the hit also depends on the block + # holding that unit -- a different block when the key ends a block + # (and never part of the returned hit). Pin it too. + unit = self.block_pool.hash_block_size + proof_idx = num_hit // unit + if proof_idx < len(request.block_hashes): + seen = {block.block_id for block in blocks} + for group_id in range(self.num_kv_cache_groups): + cached = self.block_pool.get_cached_block( + request.block_hashes[proof_idx], [group_id] + ) + if cached and cached[0].block_id not in seen: + seen.add(cached[0].block_id) + blocks.append(cached[0]) + self.block_pool.touch(blocks) + self._pinned_hits[request_id] = blocks + return len(blocks) + + def pinned_hit_blocks(self, request_id: str) -> list[KVCacheBlock]: + """The blocks pinned for ``request_id`` (empty if none).""" + return list(self._pinned_hits.get(request_id, ())) + + def release_pinned_hit(self, request_id: str) -> int: + """Drop the request's pin (admission, abort, finish). Returns the + number of blocks released.""" + blocks = self._pinned_hits.pop(request_id, None) + if not blocks: + return 0 + # Reversed: the prefix stays an eviction candidate longer than the + # tail, as after a free. + self.block_pool.free_blocks(reversed(blocks)) + return len(blocks) + + def yield_pinned_hits( + self, keep: str | None = None, limit: int | None = None + ) -> int: + """Drop pins (except ``keep``'s) so a running request can allocate, + or a waiter ahead in line can be admitted: the most recently pinned + waiters first, at most ``limit`` of them per call (all by default; + the scheduler passes 1 and retries, so a decode step that needs a + block costs one waiter its pin, not every waiter). Returns the number + of blocks released; 0 means there was nothing to yield.""" + released = 0 + yielded = 0 + for request_id in reversed(list(self._pinned_hits)): + if request_id == keep: + continue + if limit is not None and yielded >= limit: + break + released += self.release_pinned_hit(request_id) + yielded += 1 + return released + def remove_skipped_blocks( self, request_id: str, @@ -683,6 +793,8 @@ class KVCacheManager: False otherwise. """ + # Pinned waiters' hits are about to be invalidated anyway. + self.yield_pinned_hits() if not self.coordinator.reset_prefix_cache(): return False if self.log_stats: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 4d5e490ce5..671417a105 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -372,6 +372,8 @@ class Scheduler(SchedulerInterface): # whatever needs_kv_cache_zeroing says, like the clear itself. self._skip_ring_clear_block_ids: set[int] = set() + self._log_prefix_cache_features() + self.use_pp = self.parallel_config.pipeline_parallel_size > 1 self.use_v2_model_runner = vllm_config.use_v2_model_runner # Scheduler iteration counter. Drives the V2+PP+async decode-throttle @@ -569,6 +571,45 @@ class Scheduler(SchedulerInterface): end = min((s for s in stops if start < s < end), default=end) return max(end - start, 0) + def _log_prefix_cache_features(self) -> None: + """One startup line per prefix-cache retention feature that is active + (nothing when it is off).""" + manager = self.kv_cache_manager + if manager.enable_caching and self.connector is None: + logger.info( + "Prefix cache: waiting requests' cached prefixes are pinned " + "until admission (up to %d waiters; hash unit %d, block %d)", + manager.MAX_PINNED_WAITERS, + self.hash_block_size, + self.block_size, + ) + + def _pin_waiting_hits(self) -> None: + """Pin the prefix-cache hits of the requests at the front of the + waiting queue (arrival order, bounded by the manager) so the + admissions and decode growth of other requests cannot evict them + while they wait -- also while the running set is full, when they are + not even looked up. Released on admission, free, and whenever a + running request would otherwise fail to allocate. + """ + manager = self.kv_cache_manager + if ( + not manager.enable_caching + or self.connector is not None + or manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS + ): + return + for request in self.waiting: + if manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS: + break + if ( + request.num_computed_tokens != 0 + or request.status + not in (RequestStatus.WAITING, RequestStatus.PREEMPTED) + ): + continue + manager.pin_waiting_hit(request) + def _get_local_prefix_cache_hit( self, request: Request ) -> tuple[KVCacheBlocks, int, int, bool]: @@ -803,6 +844,12 @@ class Scheduler(SchedulerInterface): ): break + # Waiting requests' pinned prefix hits yield + # before anyone is preempted (a pin never costs a running + # request its blocks); retry with them in the free queue. + if self.kv_cache_manager.yield_pinned_hits(limit=1): + continue + # The request cannot be scheduled. # Preempt the lowest-priority request. if self.policy == SchedulingPolicy.PRIORITY: @@ -1279,6 +1326,26 @@ class Scheduler(SchedulerInterface): has_scheduled_reqs=bool(self.running), ) + if new_blocks is None and self.kv_cache_manager.yield_pinned_hits( + keep=request_id + ): + # Pins of waiters behind this one yield to it + # (arrival order); its own pin stays, those blocks are + # its hit. One retry with them back in the free queue. + new_blocks = self.kv_cache_manager.allocate_slots( + request, + num_new_tokens, + num_new_computed_tokens=num_new_local_computed_tokens, + new_computed_blocks=new_computed_blocks, + num_lookahead_tokens=effective_lookahead_tokens, + num_external_computed_tokens=num_external_computed_tokens, + delay_cache_blocks=load_kv_async, + num_encoder_tokens=num_encoder_tokens, + full_sequence_must_fit=self.scheduler_reserve_full_isl, + reserved_blocks=reserved_blocks, + has_scheduled_reqs=bool(self.running), + ) + if new_blocks is None: # The request cannot be scheduled. @@ -1401,6 +1468,9 @@ class Scheduler(SchedulerInterface): if step_skipped_waiting: self.waiting.prepend_requests(step_skipped_waiting) + # Whoever is still waiting keeps its prefix-cache hit. + self._pin_waiting_hits() + # DP prefill balancing: on a step that admitted prefills (release), # record whether it was capacity-bound. if not defer_prefills: