From 8950394e0a40a781c61a9ef1f099ef847af03891 Mon Sep 17 00:00:00 2001 From: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Date: Tue, 21 Jul 2026 07:02:03 -0700 Subject: [PATCH] [Bugfix] Prefix-cache metrics double-counted when a KV connector defers requests (#48860) Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Signed-off-by: Seiji Eicher Signed-off-by: Nick Hill Co-authored-by: Lixi Zhang Co-authored-by: harjothkhara Co-authored-by: Claude Opus 4.8 Co-authored-by: Nick Hill --- tests/v1/core/test_scheduler.py | 135 ++++++++++++++++++++++++++++ tests/v1/core/utils.py | 11 ++- tests/v1/kv_connector/unit/utils.py | 10 +++ vllm/v1/core/kv_cache_manager.py | 25 ++++-- vllm/v1/core/sched/scheduler.py | 64 ++++++------- 5 files changed, 204 insertions(+), 41 deletions(-) diff --git a/tests/v1/core/test_scheduler.py b/tests/v1/core/test_scheduler.py index 408de40928..5cbbef1df5 100644 --- a/tests/v1/core/test_scheduler.py +++ b/tests/v1/core/test_scheduler.py @@ -1068,6 +1068,141 @@ def test_preempt_during_execution(): assert requests[1].output_token_ids[0] == 42 +def test_prefix_cache_query_not_inflated_by_connector_defer(): + """The GPU prefix-cache query is recorded at admission, so a request the + connector defers several times is counted once, not once per retry.""" + num_defers_before_matching = 3 + scheduler = create_scheduler( + enable_prefix_caching=True, + use_kv_connector=mock_kv( + matched_tokens=0, + is_async=False, + num_defers_before_matching=num_defers_before_matching, + ), + ) + request = create_requests(num_requests=1, num_tokens=32, block_size=16)[0] + scheduler.add_request(request) + + # Each deferred step re-runs the lookup but records nothing. + for _ in range(num_defers_before_matching): + assert not scheduler.schedule().scheduled_new_reqs + + output = scheduler.schedule() + assert any(r.req_id == request.request_id for r in output.scheduled_new_reqs) + + stats = scheduler.kv_cache_manager.prefix_cache_stats + assert stats is not None + assert stats.requests == 1 + assert stats.queries == request.num_tokens + + +def test_preemption_re_records_prefix_cache_query(): + """A preempted request re-enters the lookup on resume, so its recomputation + is counted again into the preempted stats.""" + scheduler = create_scheduler(enable_prefix_caching=True) + request = create_requests(num_requests=1)[0] + scheduler.add_request(request) + + scheduler.schedule() + stats = scheduler.kv_cache_manager.prefix_cache_stats + assert stats is not None + assert (stats.requests, stats.preempted_requests) == (1, 0) + + scheduler.running.remove(request) + scheduler._preempt_request(request, 0.0) + assert request.status == RequestStatus.PREEMPTED + + scheduler.schedule() + assert request.status == RequestStatus.RUNNING + assert stats.preempted_requests == 1 + + +def test_prefix_cache_stats_not_recorded_when_caching_disabled(): + """With prefix caching off there is no local lookup, so admitting a request + records no phantom miss.""" + scheduler = create_scheduler(enable_prefix_caching=False) + for request in create_requests(num_requests=2): + scheduler.add_request(request) + + scheduler.schedule() + + stats = scheduler.kv_cache_manager.prefix_cache_stats + assert stats is not None + assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0) + + +def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request(): + """A real cache hit rejected once by allocate_slots and admitted on the next + step is counted exactly once, hits included.""" + block_size = 16 + scheduler = create_scheduler( + enable_prefix_caching=True, + enable_chunked_prefill=False, + block_size=block_size, + ) + + # Seed the cache so the next request with the same prompt hits it. + seed = create_requests( + num_requests=1, + num_tokens=block_size * 2, + max_tokens=2, + same_prompt=True, + block_size=block_size, + req_ids=["seed"], + )[0] + scheduler.add_request(seed) + _step_until_done( + scheduler, + scheduler.schedule(), + ModelRunnerOutput( + req_ids=["seed"], + req_id_to_index={"seed": 0}, + sampled_token_ids=[[1000]], + logprobs=None, + prompt_logprobs_dict={}, + pooler_output=[], + ), + ) + + # The seeding step swapped in a fresh accumulator, so re-read it; the + # retried request must be the only thing recorded from here on. + stats = scheduler.kv_cache_manager.prefix_cache_stats + assert stats is not None + assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0) + + retried = create_requests( + num_requests=1, + num_tokens=block_size * 3, + max_tokens=1, + same_prompt=True, + block_size=block_size, + req_ids=["retried"], + )[0] + scheduler.add_request(retried) + + # Reject the first allocation attempt, then delegate to the real one. + orig_allocate_slots = scheduler.kv_cache_manager.allocate_slots + allocate_results: list = [] + + def spy_allocate_slots(*args, **kwargs): + result = None if not allocate_results else orig_allocate_slots(*args, **kwargs) + allocate_results.append(result) + return result + + scheduler.kv_cache_manager.allocate_slots = spy_allocate_slots + + assert not scheduler.schedule().scheduled_new_reqs + assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0) + + assert "retried" in scheduler.schedule().num_scheduled_tokens + assert allocate_results[0] is None and allocate_results[1] is not None + assert (stats.requests, stats.queries, stats.hits) == ( + 1, + retried.num_tokens, + block_size * 2, + ) + + def test_scheduler_reset_prefix_cache(): scheduler = create_scheduler(enable_prefix_caching=True) requests = create_requests(num_requests=10) diff --git a/tests/v1/core/utils.py b/tests/v1/core/utils.py index 19beba1a53..2b2d241f57 100644 --- a/tests/v1/core/utils.py +++ b/tests/v1/core/utils.py @@ -38,8 +38,12 @@ from vllm.v1.structured_output import StructuredOutputManager EOS_TOKEN_ID = 50256 -def mock_kv(matched_tokens: int, is_async: bool): - return MockKVConfig(matched_tokens=matched_tokens, is_async=is_async) +def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int = 0): + return MockKVConfig( + matched_tokens=matched_tokens, + is_async=is_async, + num_defers_before_matching=num_defers_before_matching, + ) def create_scheduler( @@ -115,6 +119,9 @@ def create_scheduler( kv_connector_extra_config={ "matched_tokens": use_kv_connector.matched_tokens, "is_async": use_kv_connector.is_async, + "num_defers_before_matching": ( + use_kv_connector.num_defers_before_matching + ), }, ) elif isinstance(use_kv_connector, str): diff --git a/tests/v1/kv_connector/unit/utils.py b/tests/v1/kv_connector/unit/utils.py index 7df9e20e6a..7d4b17ab39 100644 --- a/tests/v1/kv_connector/unit/utils.py +++ b/tests/v1/kv_connector/unit/utils.py @@ -362,6 +362,7 @@ class TestExampleConnector(ExampleConnector): class MockKVConfig: matched_tokens: int = 0 is_async: bool = False + num_defers_before_matching: int = 0 class MockKVConnectorMetadata(KVConnectorMetadata): @@ -384,6 +385,12 @@ class MockKVConnector(KVConnectorBase_V1): self.config = MockKVConfig( matched_tokens=extra_config["matched_tokens"], is_async=extra_config["is_async"], + num_defers_before_matching=extra_config.get( + "num_defers_before_matching", 0 + ), + ) + self._defers_left: defaultdict[str, int] = defaultdict( + lambda: self.config.num_defers_before_matching ) def get_num_new_matched_tokens( @@ -391,6 +398,9 @@ class MockKVConnector(KVConnectorBase_V1): request: Request, num_computed_tokens: int, ) -> tuple[int | None, bool]: + if self._defers_left[request.request_id] > 0: + self._defers_left[request.request_id] -= 1 + return (None, False) return (self.config.matched_tokens, self.config.is_async) def update_state_after_alloc( diff --git a/vllm/v1/core/kv_cache_manager.py b/vllm/v1/core/kv_cache_manager.py index 568a2774b2..cc269e6ff1 100644 --- a/vllm/v1/core/kv_cache_manager.py +++ b/vllm/v1/core/kv_cache_manager.py @@ -204,6 +204,21 @@ class KVCacheManager: self.prefix_cache_stats = PrefixCacheStats() return stats + def prefix_cache_lookup_enabled(self, request: Request) -> bool: + """Whether a local prefix cache lookup may be run for this request.""" + return self.enable_caching and not request.skip_reading_prefix_cache + + def record_prefix_cache_stats(self, request: Request, num_hits: int) -> None: + # Don't count a request that skipped the cache lookup. + if not self.log_stats or not self.prefix_cache_lookup_enabled(request): + return + assert self.prefix_cache_stats is not None + self.prefix_cache_stats.record( + num_tokens=request.num_tokens, + num_hits=num_hits, + preempted=request.num_preemptions > 0, + ) + def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int]: """Get the computed (cached) blocks for the request. Note that the computed blocks must be full. @@ -225,7 +240,7 @@ class KVCacheManager: # disabled or the request is marked as skipping kv cache read # (which happens when the request requires prompt logprobs # or calls a pooling model with all pooling). - if not self.enable_caching or request.skip_reading_prefix_cache: + if not self.prefix_cache_lookup_enabled(request): return self.empty_kv_cache_blocks, 0, 0 # NOTE: When all tokens hit the cache, we must recompute the last token @@ -269,14 +284,6 @@ class KVCacheManager: num_new_computed_tokens + num_uncached if num_uncached else 0 ) - if self.log_stats: - assert self.prefix_cache_stats is not None - self.prefix_cache_stats.record( - num_tokens=request.num_tokens, - num_hits=num_new_computed_tokens, - preempted=request.num_preemptions > 0, - ) - blocks = self.create_kv_cache_blocks(computed_blocks) return blocks, num_new_computed_tokens, shared_prefix_boundary diff --git a/vllm/v1/core/sched/scheduler.py b/vllm/v1/core/sched/scheduler.py index 46a999ef7c..a693061d51 100644 --- a/vllm/v1/core/sched/scheduler.py +++ b/vllm/v1/core/sched/scheduler.py @@ -706,46 +706,47 @@ class Scheduler(SchedulerInterface): num_external_computed_tokens = 0 load_kv_async = False connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0 + did_prefix_cache_lookup = False # Get already-cached tokens. if request.num_computed_tokens == 0: + did_prefix_cache_lookup = True # Get locally-cached tokens. if ( self.connector is not None and self.has_mamba_layers and isinstance( - self.kv_cache_manager.coordinator, - HybridKVCacheCoordinator, + self.kv_cache_manager.coordinator, HybridKVCacheCoordinator ) ): - computed, per_group_hits = ( - self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group( - request.block_hashes, request.num_tokens - 1 - ) - ) - new_computed_blocks = ( - self.kv_cache_manager.create_kv_cache_blocks(computed) - ) - # NOTE(ZhanqiuHu): For Mamba hybrid models, - # num_new_local_computed_tokens should be the FA hit - # length. This value is passed to the connector's - # get_num_new_matched_tokens which computes: - # external = total - local_computed. - # Using the FA hit skips re-transferring FA blocks - # already cached on D-side. The Mamba state (always - # the last block) is transferred unconditionally by - # _apply_prefix_caching in nixl/worker.py. - num_new_local_computed_tokens = max(per_group_hits) # The per-group lookup does not detect an uncached shared # prefix, so there is no junction to pin in this path. request.shared_prefix_boundary = 0 - if self.kv_cache_manager.log_stats: - assert self.kv_cache_manager.prefix_cache_stats is not None - self.kv_cache_manager.prefix_cache_stats.record( - num_tokens=request.num_tokens, - num_hits=num_new_local_computed_tokens, - preempted=request.num_preemptions > 0, + kv_cache_manager = self.kv_cache_manager + if not kv_cache_manager.prefix_cache_lookup_enabled(request): + # Mirror the get_computed_blocks() early-out: the + # request must recompute its prompt. + new_computed_blocks = kv_cache_manager.empty_kv_cache_blocks + num_new_local_computed_tokens = 0 + else: + computed, per_group_hits = ( + self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group( + request.block_hashes, request.num_tokens - 1 + ) ) + new_computed_blocks = ( + self.kv_cache_manager.create_kv_cache_blocks(computed) + ) + # NOTE(ZhanqiuHu): For Mamba hybrid models, + # num_new_local_computed_tokens should be the FA hit + # length. This value is passed to the connector's + # get_num_new_matched_tokens which computes: + # external = total - local_computed. + # Using the FA hit skips re-transferring FA blocks + # already cached on D-side. The Mamba state (always + # the last block) is transferred unconditionally by + # _apply_prefix_caching in nixl/worker.py. + num_new_local_computed_tokens = max(per_group_hits) else: ( new_computed_blocks, @@ -799,10 +800,7 @@ class Scheduler(SchedulerInterface): continue # Track first scheduled prefill, not post-preemption repeat prefills - if ( - request.prefill_stats is not None - and request.num_preemptions <= 0 - ): + if request.prefill_stats and request.num_preemptions <= 0: assert num_computed_tokens <= request.num_prompt_tokens request.prefill_stats.set( num_prompt_tokens=request.num_prompt_tokens, @@ -971,6 +969,12 @@ class Scheduler(SchedulerInterface): preempted=request.num_preemptions > 0, ) + # Record at admission so unscheduled lookups are not counted. + if did_prefix_cache_lookup: + self.kv_cache_manager.record_prefix_cache_stats( + request, num_new_local_computed_tokens + ) + request = request_queue.pop_request() if load_kv_async: # If loading async, allocate memory and put request