[KV Offload] Split tiering_lookup_delay into sync/async histograms (#47679)

Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com>
Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
Srinivas Krovvidi
2026-07-16 20:03:45 +03:00
committed by GitHub
co-authored by srinivas_oo7
parent 7d56fe2adc
commit ce65385618
4 changed files with 208 additions and 5 deletions
@@ -35,6 +35,7 @@ from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
SecondaryTierManager,
TieringOffloadingMetrics,
)
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
@@ -233,9 +234,9 @@ class TestTieringOffloadingManager:
secondary_tiers=[self.secondary_tier1, self.secondary_tier2],
)
def _simulate_on_schedule_end(self):
def _simulate_on_schedule_end(self, new_req_ids: list[str] | None = None):
"""Simulate end of scheduler step: lifecycle flush + drain events."""
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
ctx = ScheduleEndContext(new_req_ids=new_req_ids or [], preempted_req_ids=())
self.manager.on_schedule_end(ctx)
list(self.manager.take_events())
@@ -395,6 +396,87 @@ class TestTieringOffloadingManager:
# Next lookup should succeed
assert count_hits(self.manager, blocks) == 3
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
"""Resolved lookups report one sync delay sample on allocation."""
self._start_request()
blocks = to_keys(range(2))
# No tier has these blocks: they resolve immediately as misses.
for block in blocks:
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
stats.reduce()
)
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced
def test_lookup_reports_async_delay_across_promotion(self, manager_setup):
"""A new request reports async delay at schedule end."""
self._start_request()
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
# First lookup finds the block in a secondary tier and defers.
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
# The first scheduler step reports async delay for the new request.
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
# Promotion completes on the next scheduler step.
self._simulate_on_schedule_end()
# Next lookup resolves via the now-promoted primary-tier block.
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
stats.reduce()
)
def test_lookup_reports_async_delay_on_request_finish(self, manager_setup):
"""Never-allocated lookup delays flush at teardown."""
ctx = ReqContext(req_id="req_lookup_finish")
self._start_request(ctx)
block = to_keys(range(1))[0]
self.secondary_tier1.blocks[block] = True
# Lookup finds the block in a secondary tier and defers.
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
self._simulate_on_schedule_end()
stats = self.manager.get_stats()
if stats is not None:
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
stats.reduce()
)
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
stats.reduce()
)
# Request finishes before the deferred lookup is ever resolved.
self.manager.on_request_finished(ctx)
stats = self.manager.get_stats()
assert stats is not None
reduced = stats.reduce()
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
def test_partial_lookup(self, manager_setup):
"""Test lookup with partial hits."""
blocks = to_keys(range(5))
+7
View File
@@ -31,6 +31,13 @@ if TYPE_CHECKING:
JobId = int
class TieringOffloadingMetrics:
"""Metric names for TieringOffloadingManager."""
LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds"
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds"
@dataclass
class JobMetadata:
"""Metadata for an in-flight async transfer job."""
+71 -3
View File
@@ -20,6 +20,7 @@ Key Design Principles:
protecting blocks from eviction until complete_read() is called
"""
import time
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass, field
@@ -50,6 +51,7 @@ from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
ParentManager,
SecondaryTierManager,
TieringOffloadingMetrics,
)
logger = init_logger(__name__)
@@ -70,6 +72,10 @@ class RequestState:
pending_primary_stores: int = 0
is_finished: bool = False
request_level_tiers: set[SecondaryTierManager] | None = None
sync_lookup_delay: float = 0.0
# time.monotonic() of this request's first deferred secondary-tier lookup;
# None once consumed (observed) or while no secondary lookup is pending.
secondary_lookup_start_time: float | None = None
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
@@ -212,6 +218,10 @@ class TieringOffloadingManager(OffloadingManager):
for tier in self.secondary_tiers
}
# Buffers manager-level observations (e.g. lookup delay) between
# get_stats() calls; merged in and reset each time get_stats() runs.
self._stats = OffloadingConnectorStats()
def _next_job_id(self) -> JobId:
"""Generate a unique job ID for async transfer tracking."""
job_id = self._job_id_counter
@@ -301,28 +311,68 @@ class TieringOffloadingManager(OffloadingManager):
# in time for a promotion this lookup may initiate.
self._maybe_process_finished_jobs()
req_state = self._req_state.get(req_context.req_id)
primary_hit = self.primary_tier.lookup(key, req_context)
if primary_hit is LookupResult.HIT:
return LookupResult.HIT
if primary_hit is LookupResult.HIT_PENDING:
return LookupResult.HIT_PENDING
lookup_start = time.monotonic()
any_retry = False
for tier in self.secondary_tiers:
if tier is exclude_tier:
continue
result = tier.lookup(key, req_context)
if result is LookupResult.HIT:
if not self._initiate_promotion(tier, key, req_context):
return LookupResult.MISS
return LookupResult.RETRY
promoted = self._initiate_promotion(tier, key, req_context)
self._accumulate_lookup_sync_delay(req_state, lookup_start)
if (
req_state is not None
and promoted
and req_state.secondary_lookup_start_time is None
):
req_state.secondary_lookup_start_time = lookup_start
return LookupResult.MISS if not promoted else LookupResult.RETRY
if result is LookupResult.RETRY:
any_retry = True
self._accumulate_lookup_sync_delay(req_state, lookup_start)
if any_retry:
if req_state is not None and req_state.secondary_lookup_start_time is None:
req_state.secondary_lookup_start_time = lookup_start
return LookupResult.RETRY
return LookupResult.MISS
def _accumulate_lookup_sync_delay(
self, req_state: RequestState | None, start_time: float
) -> None:
"""Accumulate secondary-tier lookup time until allocation or finish."""
if req_state is not None:
req_state.sync_lookup_delay += time.monotonic() - start_time
def _maybe_observe_lookup_sync_delay(self, req_state: RequestState) -> None:
delay = req_state.sync_lookup_delay
if delay == 0:
return
req_state.sync_lookup_delay = 0.0
self._stats.observe_histogram(
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY,
delay,
)
def _maybe_observe_lookup_async_delay(self, req_state: RequestState) -> None:
"""Flush a pending deferred secondary-tier lookup timer, if any."""
start_time = req_state.secondary_lookup_start_time
if start_time is None:
return
req_state.secondary_lookup_start_time = None
self._stats.observe_histogram(
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY,
time.monotonic() - start_time,
)
def _initiate_promotion(
self,
tier: SecondaryTierManager,
@@ -666,6 +716,8 @@ class TieringOffloadingManager(OffloadingManager):
if tier is exclude_tier:
continue
tier.on_request_finished(state.req_context)
self._maybe_observe_lookup_sync_delay(state)
self._maybe_observe_lookup_async_delay(state)
del self._req_state[req_id]
@override
@@ -692,6 +744,13 @@ class TieringOffloadingManager(OffloadingManager):
for tier in self.secondary_tiers:
tier.on_schedule_end(context)
for req_id in context.new_req_ids:
state = self._req_state.get(req_id)
if state is None:
continue
self._maybe_observe_lookup_sync_delay(state)
self._maybe_observe_lookup_async_delay(state)
@override
def has_pending_work(self) -> bool:
# In-flight primary<->secondary transfers (pending promotions are
@@ -745,6 +804,8 @@ class TieringOffloadingManager(OffloadingManager):
continue
for tier in self.secondary_tiers:
tier.on_request_finished(state.req_context)
self._maybe_observe_lookup_sync_delay(state)
self._maybe_observe_lookup_async_delay(state)
finished_req_ids.append(req_id)
self.primary_tier.reset_cache()
@@ -769,6 +830,13 @@ class TieringOffloadingManager(OffloadingManager):
else:
stats.aggregate(tier_stats)
if not self._stats.is_empty():
if stats is None:
stats = self._stats
else:
stats.aggregate(self._stats)
self._stats = OffloadingConnectorStats()
return stats
@override
+46
View File
@@ -39,6 +39,7 @@ from typing_extensions import override
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
CanonicalKVCaches,
OffloadingHistogramMetadata,
OffloadingManager,
OffloadingMetricMetadata,
)
@@ -46,6 +47,7 @@ from vllm.v1.kv_offload.config import OffloadingConfig
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
from vllm.v1.kv_offload.tiering.manager import (
CPUPrimaryTierOffloadingManager,
@@ -76,6 +78,50 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
cls, extra_config: dict[str, Any]
) -> dict[str, OffloadingMetricMetadata]:
metrics = super().build_metric_definitions(extra_config)
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
OffloadingHistogramMetadata(
documentation=(
"Histogram of total blocking time spent querying secondary "
"tiers for a request, accumulated from first lookup until "
"the request is allocated or finishes, in seconds."
),
buckets=(
0.00001,
0.00005,
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
),
)
)
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
OffloadingHistogramMetadata(
documentation=(
"Histogram of wall-clock time from a request's first deferred "
"secondary-tier lookup until the request is allocated or "
"finishes, in seconds."
),
buckets=(
0.0001,
0.0005,
0.001,
0.005,
0.01,
0.05,
0.1,
0.5,
1,
5,
10,
),
)
)
secondary_tier_configs = extra_config.get("secondary_tiers", [])
if not isinstance(secondary_tier_configs, list):
raise ValueError("secondary_tiers must be a list of tier configurations")