[Bugfix] Prefix-cache metrics double-counted when a KV connector defers requests (#48860)

Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com>
Signed-off-by: Seiji Eicher <seiji@anyscale.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Lixi Zhang <muxixibbb@gmail.com>
Co-authored-by: harjothkhara <harjoth.khara@gmail.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Seiji Eicher
2026-07-21 15:02:03 +01:00
committed by GitHub
co-authored by Lixi Zhang harjothkhara Claude Opus 4.8 Nick Hill
parent 7bb49be4d1
commit 8950394e0a
5 changed files with 204 additions and 41 deletions
+135
View File
@@ -1068,6 +1068,141 @@ def test_preempt_during_execution():
assert requests[1].output_token_ids[0] == 42
def test_prefix_cache_query_not_inflated_by_connector_defer():
"""The GPU prefix-cache query is recorded at admission, so a request the
connector defers several times is counted once, not once per retry."""
num_defers_before_matching = 3
scheduler = create_scheduler(
enable_prefix_caching=True,
use_kv_connector=mock_kv(
matched_tokens=0,
is_async=False,
num_defers_before_matching=num_defers_before_matching,
),
)
request = create_requests(num_requests=1, num_tokens=32, block_size=16)[0]
scheduler.add_request(request)
# Each deferred step re-runs the lookup but records nothing.
for _ in range(num_defers_before_matching):
assert not scheduler.schedule().scheduled_new_reqs
output = scheduler.schedule()
assert any(r.req_id == request.request_id for r in output.scheduled_new_reqs)
stats = scheduler.kv_cache_manager.prefix_cache_stats
assert stats is not None
assert stats.requests == 1
assert stats.queries == request.num_tokens
def test_preemption_re_records_prefix_cache_query():
"""A preempted request re-enters the lookup on resume, so its recomputation
is counted again into the preempted stats."""
scheduler = create_scheduler(enable_prefix_caching=True)
request = create_requests(num_requests=1)[0]
scheduler.add_request(request)
scheduler.schedule()
stats = scheduler.kv_cache_manager.prefix_cache_stats
assert stats is not None
assert (stats.requests, stats.preempted_requests) == (1, 0)
scheduler.running.remove(request)
scheduler._preempt_request(request, 0.0)
assert request.status == RequestStatus.PREEMPTED
scheduler.schedule()
assert request.status == RequestStatus.RUNNING
assert stats.preempted_requests == 1
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
"""With prefix caching off there is no local lookup, so admitting a request
records no phantom miss."""
scheduler = create_scheduler(enable_prefix_caching=False)
for request in create_requests(num_requests=2):
scheduler.add_request(request)
scheduler.schedule()
stats = scheduler.kv_cache_manager.prefix_cache_stats
assert stats is not None
assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0)
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
"""A real cache hit rejected once by allocate_slots and admitted on the next
step is counted exactly once, hits included."""
block_size = 16
scheduler = create_scheduler(
enable_prefix_caching=True,
enable_chunked_prefill=False,
block_size=block_size,
)
# Seed the cache so the next request with the same prompt hits it.
seed = create_requests(
num_requests=1,
num_tokens=block_size * 2,
max_tokens=2,
same_prompt=True,
block_size=block_size,
req_ids=["seed"],
)[0]
scheduler.add_request(seed)
_step_until_done(
scheduler,
scheduler.schedule(),
ModelRunnerOutput(
req_ids=["seed"],
req_id_to_index={"seed": 0},
sampled_token_ids=[[1000]],
logprobs=None,
prompt_logprobs_dict={},
pooler_output=[],
),
)
# The seeding step swapped in a fresh accumulator, so re-read it; the
# retried request must be the only thing recorded from here on.
stats = scheduler.kv_cache_manager.prefix_cache_stats
assert stats is not None
assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0)
retried = create_requests(
num_requests=1,
num_tokens=block_size * 3,
max_tokens=1,
same_prompt=True,
block_size=block_size,
req_ids=["retried"],
)[0]
scheduler.add_request(retried)
# Reject the first allocation attempt, then delegate to the real one.
orig_allocate_slots = scheduler.kv_cache_manager.allocate_slots
allocate_results: list = []
def spy_allocate_slots(*args, **kwargs):
result = None if not allocate_results else orig_allocate_slots(*args, **kwargs)
allocate_results.append(result)
return result
scheduler.kv_cache_manager.allocate_slots = spy_allocate_slots
assert not scheduler.schedule().scheduled_new_reqs
assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0)
assert "retried" in scheduler.schedule().num_scheduled_tokens
assert allocate_results[0] is None and allocate_results[1] is not None
assert (stats.requests, stats.queries, stats.hits) == (
1,
retried.num_tokens,
block_size * 2,
)
def test_scheduler_reset_prefix_cache():
scheduler = create_scheduler(enable_prefix_caching=True)
requests = create_requests(num_requests=10)
+9 -2
View File
@@ -38,8 +38,12 @@ from vllm.v1.structured_output import StructuredOutputManager
EOS_TOKEN_ID = 50256
def mock_kv(matched_tokens: int, is_async: bool):
return MockKVConfig(matched_tokens=matched_tokens, is_async=is_async)
def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int = 0):
return MockKVConfig(
matched_tokens=matched_tokens,
is_async=is_async,
num_defers_before_matching=num_defers_before_matching,
)
def create_scheduler(
@@ -115,6 +119,9 @@ def create_scheduler(
kv_connector_extra_config={
"matched_tokens": use_kv_connector.matched_tokens,
"is_async": use_kv_connector.is_async,
"num_defers_before_matching": (
use_kv_connector.num_defers_before_matching
),
},
)
elif isinstance(use_kv_connector, str):
+10
View File
@@ -362,6 +362,7 @@ class TestExampleConnector(ExampleConnector):
class MockKVConfig:
matched_tokens: int = 0
is_async: bool = False
num_defers_before_matching: int = 0
class MockKVConnectorMetadata(KVConnectorMetadata):
@@ -384,6 +385,12 @@ class MockKVConnector(KVConnectorBase_V1):
self.config = MockKVConfig(
matched_tokens=extra_config["matched_tokens"],
is_async=extra_config["is_async"],
num_defers_before_matching=extra_config.get(
"num_defers_before_matching", 0
),
)
self._defers_left: defaultdict[str, int] = defaultdict(
lambda: self.config.num_defers_before_matching
)
def get_num_new_matched_tokens(
@@ -391,6 +398,9 @@ class MockKVConnector(KVConnectorBase_V1):
request: Request,
num_computed_tokens: int,
) -> tuple[int | None, bool]:
if self._defers_left[request.request_id] > 0:
self._defers_left[request.request_id] -= 1
return (None, False)
return (self.config.matched_tokens, self.config.is_async)
def update_state_after_alloc(
+16 -9
View File
@@ -204,6 +204,21 @@ class KVCacheManager:
self.prefix_cache_stats = PrefixCacheStats()
return stats
def prefix_cache_lookup_enabled(self, request: Request) -> bool:
"""Whether a local prefix cache lookup may be run for this request."""
return self.enable_caching and not request.skip_reading_prefix_cache
def record_prefix_cache_stats(self, request: Request, num_hits: int) -> None:
# Don't count a request that skipped the cache lookup.
if not self.log_stats or not self.prefix_cache_lookup_enabled(request):
return
assert self.prefix_cache_stats is not None
self.prefix_cache_stats.record(
num_tokens=request.num_tokens,
num_hits=num_hits,
preempted=request.num_preemptions > 0,
)
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int]:
"""Get the computed (cached) blocks for the request.
Note that the computed blocks must be full.
@@ -225,7 +240,7 @@ class KVCacheManager:
# disabled or the request is marked as skipping kv cache read
# (which happens when the request requires prompt logprobs
# or calls a pooling model with all pooling).
if not self.enable_caching or request.skip_reading_prefix_cache:
if not self.prefix_cache_lookup_enabled(request):
return self.empty_kv_cache_blocks, 0, 0
# NOTE: When all tokens hit the cache, we must recompute the last token
@@ -269,14 +284,6 @@ class KVCacheManager:
num_new_computed_tokens + num_uncached if num_uncached else 0
)
if self.log_stats:
assert self.prefix_cache_stats is not None
self.prefix_cache_stats.record(
num_tokens=request.num_tokens,
num_hits=num_new_computed_tokens,
preempted=request.num_preemptions > 0,
)
blocks = self.create_kv_cache_blocks(computed_blocks)
return blocks, num_new_computed_tokens, shared_prefix_boundary
+34 -30
View File
@@ -706,46 +706,47 @@ class Scheduler(SchedulerInterface):
num_external_computed_tokens = 0
load_kv_async = False
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
did_prefix_cache_lookup = False
# Get already-cached tokens.
if request.num_computed_tokens == 0:
did_prefix_cache_lookup = True
# Get locally-cached tokens.
if (
self.connector is not None
and self.has_mamba_layers
and isinstance(
self.kv_cache_manager.coordinator,
HybridKVCacheCoordinator,
self.kv_cache_manager.coordinator, HybridKVCacheCoordinator
)
):
computed, per_group_hits = (
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
request.block_hashes, request.num_tokens - 1
)
)
new_computed_blocks = (
self.kv_cache_manager.create_kv_cache_blocks(computed)
)
# NOTE(ZhanqiuHu): For Mamba hybrid models,
# num_new_local_computed_tokens should be the FA hit
# length. This value is passed to the connector's
# get_num_new_matched_tokens which computes:
# external = total - local_computed.
# Using the FA hit skips re-transferring FA blocks
# already cached on D-side. The Mamba state (always
# the last block) is transferred unconditionally by
# _apply_prefix_caching in nixl/worker.py.
num_new_local_computed_tokens = max(per_group_hits)
# The per-group lookup does not detect an uncached shared
# prefix, so there is no junction to pin in this path.
request.shared_prefix_boundary = 0
if self.kv_cache_manager.log_stats:
assert self.kv_cache_manager.prefix_cache_stats is not None
self.kv_cache_manager.prefix_cache_stats.record(
num_tokens=request.num_tokens,
num_hits=num_new_local_computed_tokens,
preempted=request.num_preemptions > 0,
kv_cache_manager = self.kv_cache_manager
if not kv_cache_manager.prefix_cache_lookup_enabled(request):
# Mirror the get_computed_blocks() early-out: the
# request must recompute its prompt.
new_computed_blocks = kv_cache_manager.empty_kv_cache_blocks
num_new_local_computed_tokens = 0
else:
computed, per_group_hits = (
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
request.block_hashes, request.num_tokens - 1
)
)
new_computed_blocks = (
self.kv_cache_manager.create_kv_cache_blocks(computed)
)
# NOTE(ZhanqiuHu): For Mamba hybrid models,
# num_new_local_computed_tokens should be the FA hit
# length. This value is passed to the connector's
# get_num_new_matched_tokens which computes:
# external = total - local_computed.
# Using the FA hit skips re-transferring FA blocks
# already cached on D-side. The Mamba state (always
# the last block) is transferred unconditionally by
# _apply_prefix_caching in nixl/worker.py.
num_new_local_computed_tokens = max(per_group_hits)
else:
(
new_computed_blocks,
@@ -799,10 +800,7 @@ class Scheduler(SchedulerInterface):
continue
# Track first scheduled prefill, not post-preemption repeat prefills
if (
request.prefill_stats is not None
and request.num_preemptions <= 0
):
if request.prefill_stats and request.num_preemptions <= 0:
assert num_computed_tokens <= request.num_prompt_tokens
request.prefill_stats.set(
num_prompt_tokens=request.num_prompt_tokens,
@@ -971,6 +969,12 @@ class Scheduler(SchedulerInterface):
preempted=request.num_preemptions > 0,
)
# Record at admission so unscheduled lookups are not counted.
if did_prefix_cache_lookup:
self.kv_cache_manager.record_prefix_cache_stats(
request, num_new_local_computed_tokens
)
request = request_queue.pop_request()
if load_kv_async:
# If loading async, allocate memory and put request