[Bugfix] Prefix-cache metrics double-counted when a KV connector defers requests (#48860)
Signed-off-by: Seiji Eicher <58963096+eicherseiji@users.noreply.github.com> Signed-off-by: Seiji Eicher <seiji@anyscale.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Lixi Zhang <muxixibbb@gmail.com> Co-authored-by: harjothkhara <harjoth.khara@gmail.com> Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Lixi Zhang
harjothkhara
Claude Opus 4.8
Nick Hill
parent
7bb49be4d1
commit
8950394e0a
@@ -1068,6 +1068,141 @@ def test_preempt_during_execution():
|
||||
assert requests[1].output_token_ids[0] == 42
|
||||
|
||||
|
||||
def test_prefix_cache_query_not_inflated_by_connector_defer():
|
||||
"""The GPU prefix-cache query is recorded at admission, so a request the
|
||||
connector defers several times is counted once, not once per retry."""
|
||||
num_defers_before_matching = 3
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=mock_kv(
|
||||
matched_tokens=0,
|
||||
is_async=False,
|
||||
num_defers_before_matching=num_defers_before_matching,
|
||||
),
|
||||
)
|
||||
request = create_requests(num_requests=1, num_tokens=32, block_size=16)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Each deferred step re-runs the lookup but records nothing.
|
||||
for _ in range(num_defers_before_matching):
|
||||
assert not scheduler.schedule().scheduled_new_reqs
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert any(r.req_id == request.request_id for r in output.scheduled_new_reqs)
|
||||
|
||||
stats = scheduler.kv_cache_manager.prefix_cache_stats
|
||||
assert stats is not None
|
||||
assert stats.requests == 1
|
||||
assert stats.queries == request.num_tokens
|
||||
|
||||
|
||||
def test_preemption_re_records_prefix_cache_query():
|
||||
"""A preempted request re-enters the lookup on resume, so its recomputation
|
||||
is counted again into the preempted stats."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
request = create_requests(num_requests=1)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.schedule()
|
||||
stats = scheduler.kv_cache_manager.prefix_cache_stats
|
||||
assert stats is not None
|
||||
assert (stats.requests, stats.preempted_requests) == (1, 0)
|
||||
|
||||
scheduler.running.remove(request)
|
||||
scheduler._preempt_request(request, 0.0)
|
||||
assert request.status == RequestStatus.PREEMPTED
|
||||
|
||||
scheduler.schedule()
|
||||
assert request.status == RequestStatus.RUNNING
|
||||
assert stats.preempted_requests == 1
|
||||
|
||||
|
||||
def test_prefix_cache_stats_not_recorded_when_caching_disabled():
|
||||
"""With prefix caching off there is no local lookup, so admitting a request
|
||||
records no phantom miss."""
|
||||
scheduler = create_scheduler(enable_prefix_caching=False)
|
||||
for request in create_requests(num_requests=2):
|
||||
scheduler.add_request(request)
|
||||
|
||||
scheduler.schedule()
|
||||
|
||||
stats = scheduler.kv_cache_manager.prefix_cache_stats
|
||||
assert stats is not None
|
||||
assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0)
|
||||
|
||||
|
||||
def test_prefix_cache_stats_counted_once_for_retried_then_scheduled_request():
|
||||
"""A real cache hit rejected once by allocate_slots and admitted on the next
|
||||
step is counted exactly once, hits included."""
|
||||
block_size = 16
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
enable_chunked_prefill=False,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
# Seed the cache so the next request with the same prompt hits it.
|
||||
seed = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=block_size * 2,
|
||||
max_tokens=2,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["seed"],
|
||||
)[0]
|
||||
scheduler.add_request(seed)
|
||||
_step_until_done(
|
||||
scheduler,
|
||||
scheduler.schedule(),
|
||||
ModelRunnerOutput(
|
||||
req_ids=["seed"],
|
||||
req_id_to_index={"seed": 0},
|
||||
sampled_token_ids=[[1000]],
|
||||
logprobs=None,
|
||||
prompt_logprobs_dict={},
|
||||
pooler_output=[],
|
||||
),
|
||||
)
|
||||
|
||||
# The seeding step swapped in a fresh accumulator, so re-read it; the
|
||||
# retried request must be the only thing recorded from here on.
|
||||
stats = scheduler.kv_cache_manager.prefix_cache_stats
|
||||
assert stats is not None
|
||||
assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0)
|
||||
|
||||
retried = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=block_size * 3,
|
||||
max_tokens=1,
|
||||
same_prompt=True,
|
||||
block_size=block_size,
|
||||
req_ids=["retried"],
|
||||
)[0]
|
||||
scheduler.add_request(retried)
|
||||
|
||||
# Reject the first allocation attempt, then delegate to the real one.
|
||||
orig_allocate_slots = scheduler.kv_cache_manager.allocate_slots
|
||||
allocate_results: list = []
|
||||
|
||||
def spy_allocate_slots(*args, **kwargs):
|
||||
result = None if not allocate_results else orig_allocate_slots(*args, **kwargs)
|
||||
allocate_results.append(result)
|
||||
return result
|
||||
|
||||
scheduler.kv_cache_manager.allocate_slots = spy_allocate_slots
|
||||
|
||||
assert not scheduler.schedule().scheduled_new_reqs
|
||||
assert (stats.requests, stats.queries, stats.hits) == (0, 0, 0)
|
||||
|
||||
assert "retried" in scheduler.schedule().num_scheduled_tokens
|
||||
assert allocate_results[0] is None and allocate_results[1] is not None
|
||||
assert (stats.requests, stats.queries, stats.hits) == (
|
||||
1,
|
||||
retried.num_tokens,
|
||||
block_size * 2,
|
||||
)
|
||||
|
||||
|
||||
def test_scheduler_reset_prefix_cache():
|
||||
scheduler = create_scheduler(enable_prefix_caching=True)
|
||||
requests = create_requests(num_requests=10)
|
||||
|
||||
@@ -38,8 +38,12 @@ from vllm.v1.structured_output import StructuredOutputManager
|
||||
EOS_TOKEN_ID = 50256
|
||||
|
||||
|
||||
def mock_kv(matched_tokens: int, is_async: bool):
|
||||
return MockKVConfig(matched_tokens=matched_tokens, is_async=is_async)
|
||||
def mock_kv(matched_tokens: int, is_async: bool, num_defers_before_matching: int = 0):
|
||||
return MockKVConfig(
|
||||
matched_tokens=matched_tokens,
|
||||
is_async=is_async,
|
||||
num_defers_before_matching=num_defers_before_matching,
|
||||
)
|
||||
|
||||
|
||||
def create_scheduler(
|
||||
@@ -115,6 +119,9 @@ def create_scheduler(
|
||||
kv_connector_extra_config={
|
||||
"matched_tokens": use_kv_connector.matched_tokens,
|
||||
"is_async": use_kv_connector.is_async,
|
||||
"num_defers_before_matching": (
|
||||
use_kv_connector.num_defers_before_matching
|
||||
),
|
||||
},
|
||||
)
|
||||
elif isinstance(use_kv_connector, str):
|
||||
|
||||
@@ -362,6 +362,7 @@ class TestExampleConnector(ExampleConnector):
|
||||
class MockKVConfig:
|
||||
matched_tokens: int = 0
|
||||
is_async: bool = False
|
||||
num_defers_before_matching: int = 0
|
||||
|
||||
|
||||
class MockKVConnectorMetadata(KVConnectorMetadata):
|
||||
@@ -384,6 +385,12 @@ class MockKVConnector(KVConnectorBase_V1):
|
||||
self.config = MockKVConfig(
|
||||
matched_tokens=extra_config["matched_tokens"],
|
||||
is_async=extra_config["is_async"],
|
||||
num_defers_before_matching=extra_config.get(
|
||||
"num_defers_before_matching", 0
|
||||
),
|
||||
)
|
||||
self._defers_left: defaultdict[str, int] = defaultdict(
|
||||
lambda: self.config.num_defers_before_matching
|
||||
)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
@@ -391,6 +398,9 @@ class MockKVConnector(KVConnectorBase_V1):
|
||||
request: Request,
|
||||
num_computed_tokens: int,
|
||||
) -> tuple[int | None, bool]:
|
||||
if self._defers_left[request.request_id] > 0:
|
||||
self._defers_left[request.request_id] -= 1
|
||||
return (None, False)
|
||||
return (self.config.matched_tokens, self.config.is_async)
|
||||
|
||||
def update_state_after_alloc(
|
||||
|
||||
@@ -204,6 +204,21 @@ class KVCacheManager:
|
||||
self.prefix_cache_stats = PrefixCacheStats()
|
||||
return stats
|
||||
|
||||
def prefix_cache_lookup_enabled(self, request: Request) -> bool:
|
||||
"""Whether a local prefix cache lookup may be run for this request."""
|
||||
return self.enable_caching and not request.skip_reading_prefix_cache
|
||||
|
||||
def record_prefix_cache_stats(self, request: Request, num_hits: int) -> None:
|
||||
# Don't count a request that skipped the cache lookup.
|
||||
if not self.log_stats or not self.prefix_cache_lookup_enabled(request):
|
||||
return
|
||||
assert self.prefix_cache_stats is not None
|
||||
self.prefix_cache_stats.record(
|
||||
num_tokens=request.num_tokens,
|
||||
num_hits=num_hits,
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
|
||||
def get_computed_blocks(self, request: Request) -> tuple[KVCacheBlocks, int, int]:
|
||||
"""Get the computed (cached) blocks for the request.
|
||||
Note that the computed blocks must be full.
|
||||
@@ -225,7 +240,7 @@ class KVCacheManager:
|
||||
# disabled or the request is marked as skipping kv cache read
|
||||
# (which happens when the request requires prompt logprobs
|
||||
# or calls a pooling model with all pooling).
|
||||
if not self.enable_caching or request.skip_reading_prefix_cache:
|
||||
if not self.prefix_cache_lookup_enabled(request):
|
||||
return self.empty_kv_cache_blocks, 0, 0
|
||||
|
||||
# NOTE: When all tokens hit the cache, we must recompute the last token
|
||||
@@ -269,14 +284,6 @@ class KVCacheManager:
|
||||
num_new_computed_tokens + num_uncached if num_uncached else 0
|
||||
)
|
||||
|
||||
if self.log_stats:
|
||||
assert self.prefix_cache_stats is not None
|
||||
self.prefix_cache_stats.record(
|
||||
num_tokens=request.num_tokens,
|
||||
num_hits=num_new_computed_tokens,
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
|
||||
blocks = self.create_kv_cache_blocks(computed_blocks)
|
||||
return blocks, num_new_computed_tokens, shared_prefix_boundary
|
||||
|
||||
|
||||
@@ -706,46 +706,47 @@ class Scheduler(SchedulerInterface):
|
||||
num_external_computed_tokens = 0
|
||||
load_kv_async = False
|
||||
connector_prefix_cache_queries, connector_prefix_cache_hits = 0, 0
|
||||
did_prefix_cache_lookup = False
|
||||
|
||||
# Get already-cached tokens.
|
||||
if request.num_computed_tokens == 0:
|
||||
did_prefix_cache_lookup = True
|
||||
# Get locally-cached tokens.
|
||||
if (
|
||||
self.connector is not None
|
||||
and self.has_mamba_layers
|
||||
and isinstance(
|
||||
self.kv_cache_manager.coordinator,
|
||||
HybridKVCacheCoordinator,
|
||||
self.kv_cache_manager.coordinator, HybridKVCacheCoordinator
|
||||
)
|
||||
):
|
||||
computed, per_group_hits = (
|
||||
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
|
||||
request.block_hashes, request.num_tokens - 1
|
||||
)
|
||||
)
|
||||
new_computed_blocks = (
|
||||
self.kv_cache_manager.create_kv_cache_blocks(computed)
|
||||
)
|
||||
# NOTE(ZhanqiuHu): For Mamba hybrid models,
|
||||
# num_new_local_computed_tokens should be the FA hit
|
||||
# length. This value is passed to the connector's
|
||||
# get_num_new_matched_tokens which computes:
|
||||
# external = total - local_computed.
|
||||
# Using the FA hit skips re-transferring FA blocks
|
||||
# already cached on D-side. The Mamba state (always
|
||||
# the last block) is transferred unconditionally by
|
||||
# _apply_prefix_caching in nixl/worker.py.
|
||||
num_new_local_computed_tokens = max(per_group_hits)
|
||||
# The per-group lookup does not detect an uncached shared
|
||||
# prefix, so there is no junction to pin in this path.
|
||||
request.shared_prefix_boundary = 0
|
||||
if self.kv_cache_manager.log_stats:
|
||||
assert self.kv_cache_manager.prefix_cache_stats is not None
|
||||
self.kv_cache_manager.prefix_cache_stats.record(
|
||||
num_tokens=request.num_tokens,
|
||||
num_hits=num_new_local_computed_tokens,
|
||||
preempted=request.num_preemptions > 0,
|
||||
kv_cache_manager = self.kv_cache_manager
|
||||
if not kv_cache_manager.prefix_cache_lookup_enabled(request):
|
||||
# Mirror the get_computed_blocks() early-out: the
|
||||
# request must recompute its prompt.
|
||||
new_computed_blocks = kv_cache_manager.empty_kv_cache_blocks
|
||||
num_new_local_computed_tokens = 0
|
||||
else:
|
||||
computed, per_group_hits = (
|
||||
self.kv_cache_manager.coordinator.find_longest_cache_hit_per_group(
|
||||
request.block_hashes, request.num_tokens - 1
|
||||
)
|
||||
)
|
||||
new_computed_blocks = (
|
||||
self.kv_cache_manager.create_kv_cache_blocks(computed)
|
||||
)
|
||||
# NOTE(ZhanqiuHu): For Mamba hybrid models,
|
||||
# num_new_local_computed_tokens should be the FA hit
|
||||
# length. This value is passed to the connector's
|
||||
# get_num_new_matched_tokens which computes:
|
||||
# external = total - local_computed.
|
||||
# Using the FA hit skips re-transferring FA blocks
|
||||
# already cached on D-side. The Mamba state (always
|
||||
# the last block) is transferred unconditionally by
|
||||
# _apply_prefix_caching in nixl/worker.py.
|
||||
num_new_local_computed_tokens = max(per_group_hits)
|
||||
else:
|
||||
(
|
||||
new_computed_blocks,
|
||||
@@ -799,10 +800,7 @@ class Scheduler(SchedulerInterface):
|
||||
continue
|
||||
|
||||
# Track first scheduled prefill, not post-preemption repeat prefills
|
||||
if (
|
||||
request.prefill_stats is not None
|
||||
and request.num_preemptions <= 0
|
||||
):
|
||||
if request.prefill_stats and request.num_preemptions <= 0:
|
||||
assert num_computed_tokens <= request.num_prompt_tokens
|
||||
request.prefill_stats.set(
|
||||
num_prompt_tokens=request.num_prompt_tokens,
|
||||
@@ -971,6 +969,12 @@ class Scheduler(SchedulerInterface):
|
||||
preempted=request.num_preemptions > 0,
|
||||
)
|
||||
|
||||
# Record at admission so unscheduled lookups are not counted.
|
||||
if did_prefix_cache_lookup:
|
||||
self.kv_cache_manager.record_prefix_cache_stats(
|
||||
request, num_new_local_computed_tokens
|
||||
)
|
||||
|
||||
request = request_queue.pop_request()
|
||||
if load_kv_async:
|
||||
# If loading async, allocate memory and put request
|
||||
|
||||
Reference in New Issue
Block a user