[KV Offload] Split tiering_lookup_delay into sync/async histograms (#47679)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
Srinivas Krovvidi
2026-07-16 20:03:45 +03:00
committed by GitHub
co-authored by srinivas_oo7
parent 7d56fe2adc
commit ce65385618
4 changed files with 208 additions and 5 deletions
@@ -35,6 +35,7 @@ from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
SecondaryTierManager,
TieringOffloadingMetrics,
)
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
@@ -233,9 +234,9 @@ class TestTieringOffloadingManager:
secondary_tiers=[self.secondary_tier1, self.secondary_tier2],
)
def _simulate_on_schedule_end(self):
def _simulate_on_schedule_end(self, new_req_ids: list[str] | None = None):
"""Simulate end of scheduler step: lifecycle flush + drain events."""
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
ctx = ScheduleEndContext(new_req_ids=new_req_ids or [], preempted_req_ids=())
self.manager.on_schedule_end(ctx)
list(self.manager.take_events())
@@ -395,6 +396,87 @@ class TestTieringOffloadingManager:
# Next lookup should succeed
assert count_hits(self.manager, blocks) == 3
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample on allocation."""
self._start_request()
blocks = to_keys(range(2))
# No tier has these blocks: they resolve immediately as misses.
for block in blocks:
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
stats.reduce()
)
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced
def test_lookup_reports_async_delay_across_promotion(self, manager_setup):
"""A new request reports async delay at schedule end."""
self._start_request()
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
# First lookup finds the block in a secondary tier and defers.
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
# The first scheduler step reports async delay for the new request.
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
# Promotion completes on the next scheduler step.
self._simulate_on_schedule_end()
# Next lookup resolves via the now-promoted primary-tier block.
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
stats.reduce()
)
def test_lookup_reports_async_delay_on_request_finish(self, manager_setup):
"""Never-allocated lookup delays flush at teardown."""
ctx = ReqContext(req_id="req_lookup_finish")
self._start_request(ctx)
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
# Lookup finds the block in a secondary tier and defers.
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
self._simulate_on_schedule_end()
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
stats.reduce()
)
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
stats.reduce()
)
# Request finishes before the deferred lookup is ever resolved.
self.manager.on_request_finished(ctx)
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits."""
blocks = to_keys(range(5))