diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 4e358da2ba..c31af014bf 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -223,6 +223,21 @@ class KVCacheManager: tuple(() for _ in range(self.num_kv_cache_groups)) ) + # Prefix-cache hits pinned for requests waiting for admission + # ({request_id: blocks}). A waiting follow-up turn holds no ref on its + # cached prefix, and with a prefix unit finer than the block its + # whole hit depends on every one of those blocks (the resume key sits + # in the top block); under pool pressure other requests' allocations + # popped them from the free queue and the turn re-prefilled from 0 + # (under a long-context multi-turn agent load: 4-5 total misses per + # 300 s at unit 64/512, none at 4096). Pinned blocks stay out of the + # free queue, so they count against admission and the watermark like + # any referenced block; they + # are released on admission (the table takes over), on free, and + # whenever a running request would otherwise fail to allocate + # (``yield_pinned_hits``), so a pin never causes a preemption. + self._pinned_hits: dict[str, list[KVCacheBlock]] = {} + @property def usage(self) -> float: """Get the KV cache usage. @@ -602,6 +617,7 @@ class KVCacheManager: # P/D: delay caching blocks if we have to recv from # remote. Update state for locally cached blocks. if delay_cache_blocks: + self.release_pinned_hit(request.request_id) return self.create_kv_cache_blocks(new_blocks) # NOTE(woosuk): We want to commit (cache) up to num_local_computed_tokens @@ -615,6 +631,8 @@ class KVCacheManager: ) self.coordinator.cache_blocks(request, num_tokens_to_cache) + # Admitted: the table now references the hit, the pin is redundant. + self.release_pinned_hit(request.request_id) return self.create_kv_cache_blocks(new_blocks) def free(self, request: Request) -> None: @@ -626,8 +644,100 @@ class KVCacheManager: request: The request to free the blocks. """ + self.release_pinned_hit(request.request_id) self.coordinator.free(request.request_id) + # Waiting requests whose prefix-cache hit is pinned at once (arrival + # order). Pins never block anyone (they yield before a running request + # fails to allocate), so the bound only caps the lookup work per step. + MAX_PINNED_WAITERS = 8 + + @property + def num_pinned_waiters(self) -> int: + return len(self._pinned_hits) + + @property + def num_pinned_blocks(self) -> int: + return sum(len(blocks) for blocks in self._pinned_hits.values()) + + def pin_waiting_hit(self, request: Request) -> int: + """Pin the request's current prefix-cache hit while it waits for + admission: every non-null hit block in every group (the attention + prefix up to the key and the GDN state it resumes from) takes a ref + and leaves the free queue. Idempotent. Returns the number of blocks + pinned now (0 if nothing hit, already pinned, caching is off, or the + waiter bound is reached).""" + request_id = request.request_id + if ( + request_id in self._pinned_hits + or not self.enable_caching + or not self.prefix_cache_lookup_enabled(request) + or len(self._pinned_hits) >= self.MAX_PINNED_WAITERS + ): + return 0 + computed, num_hit, _ = self.get_computed_blocks(request) + blocks = [ + block + for group in computed.blocks + for block in group + if not block.is_null + ] + if num_hit == 0 or not blocks: + return 0 + # The EAGLE/MTP proof: fine-grained attention hits match one hash + # unit past the key and drop it, so the hit also depends on the block + # holding that unit -- a different block when the key ends a block + # (and never part of the returned hit). Pin it too. + unit = self.block_pool.hash_block_size + proof_idx = num_hit // unit + if proof_idx < len(request.block_hashes): + seen = {block.block_id for block in blocks} + for group_id in range(self.num_kv_cache_groups): + cached = self.block_pool.get_cached_block( + request.block_hashes[proof_idx], [group_id] + ) + if cached and cached[0].block_id not in seen: + seen.add(cached[0].block_id) + blocks.append(cached[0]) + self.block_pool.touch(blocks) + self._pinned_hits[request_id] = blocks + return len(blocks) + + def pinned_hit_blocks(self, request_id: str) -> list[KVCacheBlock]: + """The blocks pinned for ``request_id`` (empty if none).""" + return list(self._pinned_hits.get(request_id, ())) + + def release_pinned_hit(self, request_id: str) -> int: + """Drop the request's pin (admission, abort, finish). Returns the + number of blocks released.""" + blocks = self._pinned_hits.pop(request_id, None) + if not blocks: + return 0 + # Reversed: the prefix stays an eviction candidate longer than the + # tail, as after a free. + self.block_pool.free_blocks(reversed(blocks)) + return len(blocks) + + def yield_pinned_hits( + self, keep: str | None = None, limit: int | None = None + ) -> int: + """Drop pins (except ``keep``'s) so a running request can allocate, + or a waiter ahead in line can be admitted: the most recently pinned + waiters first, at most ``limit`` of them per call (all by default; + the scheduler passes 1 and retries, so a decode step that needs a + block costs one waiter its pin, not every waiter). Returns the number + of blocks released; 0 means there was nothing to yield.""" + released = 0 + yielded = 0 + for request_id in reversed(list(self._pinned_hits)): + if request_id == keep: + continue + if limit is not None and yielded >= limit: + break + released += self.release_pinned_hit(request_id) + yielded += 1 + return released + def remove_skipped_blocks( self, request_id: str, @@ -683,6 +793,8 @@ class KVCacheManager: False otherwise. """ + # Pinned waiters' hits are about to be invalidated anyway. + self.yield_pinned_hits() if not self.coordinator.reset_prefix_cache(): return False if self.log_stats: diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 4d5e490ce5..671417a105 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -372,6 +372,8 @@ class Scheduler(SchedulerInterface): # whatever needs_kv_cache_zeroing says, like the clear itself. self._skip_ring_clear_block_ids: set[int] = set() + self._log_prefix_cache_features() + self.use_pp = self.parallel_config.pipeline_parallel_size > 1 self.use_v2_model_runner = vllm_config.use_v2_model_runner # Scheduler iteration counter. Drives the V2+PP+async decode-throttle @@ -569,6 +571,45 @@ class Scheduler(SchedulerInterface): end = min((s for s in stops if start < s < end), default=end) return max(end - start, 0) + def _log_prefix_cache_features(self) -> None: + """One startup line per prefix-cache retention feature that is active + (nothing when it is off).""" + manager = self.kv_cache_manager + if manager.enable_caching and self.connector is None: + logger.info( + "Prefix cache: waiting requests' cached prefixes are pinned " + "until admission (up to %d waiters; hash unit %d, block %d)", + manager.MAX_PINNED_WAITERS, + self.hash_block_size, + self.block_size, + ) + + def _pin_waiting_hits(self) -> None: + """Pin the prefix-cache hits of the requests at the front of the + waiting queue (arrival order, bounded by the manager) so the + admissions and decode growth of other requests cannot evict them + while they wait -- also while the running set is full, when they are + not even looked up. Released on admission, free, and whenever a + running request would otherwise fail to allocate. + """ + manager = self.kv_cache_manager + if ( + not manager.enable_caching + or self.connector is not None + or manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS + ): + return + for request in self.waiting: + if manager.num_pinned_waiters >= manager.MAX_PINNED_WAITERS: + break + if ( + request.num_computed_tokens != 0 + or request.status + not in (RequestStatus.WAITING, RequestStatus.PREEMPTED) + ): + continue + manager.pin_waiting_hit(request) + def _get_local_prefix_cache_hit( self, request: Request ) -> tuple[KVCacheBlocks, int, int, bool]: @@ -803,6 +844,12 @@ class Scheduler(SchedulerInterface): ): break + # Waiting requests' pinned prefix hits yield + # before anyone is preempted (a pin never costs a running + # request its blocks); retry with them in the free queue. + if self.kv_cache_manager.yield_pinned_hits(limit=1): + continue + # The request cannot be scheduled. # Preempt the lowest-priority request. if self.policy == SchedulingPolicy.PRIORITY: @@ -1279,6 +1326,26 @@ class Scheduler(SchedulerInterface): has_scheduled_reqs=bool(self.running), ) + if new_blocks is None and self.kv_cache_manager.yield_pinned_hits( + keep=request_id + ): + # Pins of waiters behind this one yield to it + # (arrival order); its own pin stays, those blocks are + # its hit. One retry with them back in the free queue. + new_blocks = self.kv_cache_manager.allocate_slots( + request, + num_new_tokens, + num_new_computed_tokens=num_new_local_computed_tokens, + new_computed_blocks=new_computed_blocks, + num_lookahead_tokens=effective_lookahead_tokens, + num_external_computed_tokens=num_external_computed_tokens, + delay_cache_blocks=load_kv_async, + num_encoder_tokens=num_encoder_tokens, + full_sequence_must_fit=self.scheduler_reserve_full_isl, + reserved_blocks=reserved_blocks, + has_scheduled_reqs=bool(self.running), + ) + if new_blocks is None: # The request cannot be scheduled. @@ -1401,6 +1468,9 @@ class Scheduler(SchedulerInterface): if step_skipped_waiting: self.waiting.prepend_requests(step_skipped_waiting) + # Whoever is still waiting keeps its prefix-cache hit. + self._pin_waiting_hits() + # DP prefill balancing: on a step that admitted prefills (release), # record whether it was capacity-bound. if not defer_prefills: