diff --git a/tests/v1/kv_offload/tiering/test_tiering_offloading.py b/tests/v1/kv_offload/tiering/test_tiering_offloading.py index 13460f68d7..cf546b304c 100644 --- a/tests/v1/kv_offload/tiering/test_tiering_offloading.py +++ b/tests/v1/kv_offload/tiering/test_tiering_offloading.py @@ -35,6 +35,7 @@ from vllm.v1.kv_offload.tiering.base import ( JobMetadata, JobResult, SecondaryTierManager, + TieringOffloadingMetrics, ) from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory @@ -233,9 +234,9 @@ class TestTieringOffloadingManager: secondary_tiers=[self.secondary_tier1, self.secondary_tier2], ) - def _simulate_on_schedule_end(self): + def _simulate_on_schedule_end(self, new_req_ids: list[str] | None = None): """Simulate end of scheduler step: lifecycle flush + drain events.""" - ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=()) + ctx = ScheduleEndContext(new_req_ids=new_req_ids or [], preempted_req_ids=()) self.manager.on_schedule_end(ctx) list(self.manager.take_events()) @@ -395,6 +396,87 @@ class TestTieringOffloadingManager: # Next lookup should succeed assert count_hits(self.manager, blocks) == 3 + def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup): + """Resolved lookups report one sync delay sample on allocation.""" + self._start_request() + blocks = to_keys(range(2)) + + # No tier has these blocks: they resolve immediately as misses. + for block in blocks: + assert self.manager.lookup(block, _CTX) is LookupResult.MISS + + stats = self.manager.get_stats() + if stats is not None: + assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in ( + stats.reduce() + ) + + self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id]) + + stats = self.manager.get_stats() + assert stats is not None + reduced = stats.reduce() + assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1 + assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced + + def test_lookup_reports_async_delay_across_promotion(self, manager_setup): + """A new request reports async delay at schedule end.""" + self._start_request() + block = to_keys(range(1))[0] + self.secondary_tier1.blocks[block] = True + + # First lookup finds the block in a secondary tier and defers. + assert self.manager.lookup(block, _CTX) is LookupResult.RETRY + + # The first scheduler step reports async delay for the new request. + self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id]) + stats = self.manager.get_stats() + assert stats is not None + reduced = stats.reduce() + assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1 + assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1 + + # Promotion completes on the next scheduler step. + self._simulate_on_schedule_end() + + # Next lookup resolves via the now-promoted primary-tier block. + assert self.manager.lookup(block, _CTX) is LookupResult.HIT + + stats = self.manager.get_stats() + if stats is not None: + assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in ( + stats.reduce() + ) + + def test_lookup_reports_async_delay_on_request_finish(self, manager_setup): + """Never-allocated lookup delays flush at teardown.""" + ctx = ReqContext(req_id="req_lookup_finish") + self._start_request(ctx) + block = to_keys(range(1))[0] + self.secondary_tier1.blocks[block] = True + + # Lookup finds the block in a secondary tier and defers. + assert self.manager.lookup(block, ctx) is LookupResult.RETRY + + self._simulate_on_schedule_end() + stats = self.manager.get_stats() + if stats is not None: + assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in ( + stats.reduce() + ) + assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in ( + stats.reduce() + ) + + # Request finishes before the deferred lookup is ever resolved. + self.manager.on_request_finished(ctx) + + stats = self.manager.get_stats() + assert stats is not None + reduced = stats.reduce() + assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1 + assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1 + def test_partial_lookup(self, manager_setup): """Test lookup with partial hits.""" blocks = to_keys(range(5)) diff --git a/vllm/v1/kv_offload/tiering/base.py b/vllm/v1/kv_offload/tiering/base.py index f83113e137..f561417196 100644 --- a/vllm/v1/kv_offload/tiering/base.py +++ b/vllm/v1/kv_offload/tiering/base.py @@ -31,6 +31,13 @@ if TYPE_CHECKING: JobId = int +class TieringOffloadingMetrics: + """Metric names for TieringOffloadingManager.""" + + LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds" + LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds" + + @dataclass class JobMetadata: """Metadata for an in-flight async transfer job.""" diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index 728ac8dc86..2f5b52e222 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -20,6 +20,7 @@ Key Design Principles: protecting blocks from eviction until complete_read() is called """ +import time from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass, field @@ -50,6 +51,7 @@ from vllm.v1.kv_offload.tiering.base import ( JobMetadata, ParentManager, SecondaryTierManager, + TieringOffloadingMetrics, ) logger = init_logger(__name__) @@ -70,6 +72,10 @@ class RequestState: pending_primary_stores: int = 0 is_finished: bool = False request_level_tiers: set[SecondaryTierManager] | None = None + sync_lookup_delay: float = 0.0 + # time.monotonic() of this request's first deferred secondary-tier lookup; + # None once consumed (observed) or while no secondary lookup is pending. + secondary_lookup_start_time: float | None = None class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): @@ -212,6 +218,10 @@ class TieringOffloadingManager(OffloadingManager): for tier in self.secondary_tiers } + # Buffers manager-level observations (e.g. lookup delay) between + # get_stats() calls; merged in and reset each time get_stats() runs. + self._stats = OffloadingConnectorStats() + def _next_job_id(self) -> JobId: """Generate a unique job ID for async transfer tracking.""" job_id = self._job_id_counter @@ -301,28 +311,68 @@ class TieringOffloadingManager(OffloadingManager): # in time for a promotion this lookup may initiate. self._maybe_process_finished_jobs() + req_state = self._req_state.get(req_context.req_id) + primary_hit = self.primary_tier.lookup(key, req_context) if primary_hit is LookupResult.HIT: return LookupResult.HIT if primary_hit is LookupResult.HIT_PENDING: return LookupResult.HIT_PENDING + lookup_start = time.monotonic() any_retry = False for tier in self.secondary_tiers: if tier is exclude_tier: continue result = tier.lookup(key, req_context) if result is LookupResult.HIT: - if not self._initiate_promotion(tier, key, req_context): - return LookupResult.MISS - return LookupResult.RETRY + promoted = self._initiate_promotion(tier, key, req_context) + self._accumulate_lookup_sync_delay(req_state, lookup_start) + if ( + req_state is not None + and promoted + and req_state.secondary_lookup_start_time is None + ): + req_state.secondary_lookup_start_time = lookup_start + return LookupResult.MISS if not promoted else LookupResult.RETRY if result is LookupResult.RETRY: any_retry = True + self._accumulate_lookup_sync_delay(req_state, lookup_start) if any_retry: + if req_state is not None and req_state.secondary_lookup_start_time is None: + req_state.secondary_lookup_start_time = lookup_start return LookupResult.RETRY return LookupResult.MISS + def _accumulate_lookup_sync_delay( + self, req_state: RequestState | None, start_time: float + ) -> None: + """Accumulate secondary-tier lookup time until allocation or finish.""" + if req_state is not None: + req_state.sync_lookup_delay += time.monotonic() - start_time + + def _maybe_observe_lookup_sync_delay(self, req_state: RequestState) -> None: + delay = req_state.sync_lookup_delay + if delay == 0: + return + req_state.sync_lookup_delay = 0.0 + self._stats.observe_histogram( + TieringOffloadingMetrics.LOOKUP_SYNC_DELAY, + delay, + ) + + def _maybe_observe_lookup_async_delay(self, req_state: RequestState) -> None: + """Flush a pending deferred secondary-tier lookup timer, if any.""" + start_time = req_state.secondary_lookup_start_time + if start_time is None: + return + req_state.secondary_lookup_start_time = None + self._stats.observe_histogram( + TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY, + time.monotonic() - start_time, + ) + def _initiate_promotion( self, tier: SecondaryTierManager, @@ -666,6 +716,8 @@ class TieringOffloadingManager(OffloadingManager): if tier is exclude_tier: continue tier.on_request_finished(state.req_context) + self._maybe_observe_lookup_sync_delay(state) + self._maybe_observe_lookup_async_delay(state) del self._req_state[req_id] @override @@ -692,6 +744,13 @@ class TieringOffloadingManager(OffloadingManager): for tier in self.secondary_tiers: tier.on_schedule_end(context) + for req_id in context.new_req_ids: + state = self._req_state.get(req_id) + if state is None: + continue + self._maybe_observe_lookup_sync_delay(state) + self._maybe_observe_lookup_async_delay(state) + @override def has_pending_work(self) -> bool: # In-flight primary<->secondary transfers (pending promotions are @@ -745,6 +804,8 @@ class TieringOffloadingManager(OffloadingManager): continue for tier in self.secondary_tiers: tier.on_request_finished(state.req_context) + self._maybe_observe_lookup_sync_delay(state) + self._maybe_observe_lookup_async_delay(state) finished_req_ids.append(req_id) self.primary_tier.reset_cache() @@ -769,6 +830,13 @@ class TieringOffloadingManager(OffloadingManager): else: stats.aggregate(tier_stats) + if not self._stats.is_empty(): + if stats is None: + stats = self._stats + else: + stats.aggregate(self._stats) + self._stats = OffloadingConnectorStats() + return stats @override diff --git a/vllm/v1/kv_offload/tiering/spec.py b/vllm/v1/kv_offload/tiering/spec.py index ebf7d878d0..964699968c 100644 --- a/vllm/v1/kv_offload/tiering/spec.py +++ b/vllm/v1/kv_offload/tiering/spec.py @@ -39,6 +39,7 @@ from typing_extensions import override from vllm.logger import init_logger from vllm.v1.kv_offload.base import ( CanonicalKVCaches, + OffloadingHistogramMetadata, OffloadingManager, OffloadingMetricMetadata, ) @@ -46,6 +47,7 @@ from vllm.v1.kv_offload.config import OffloadingConfig from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec +from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory from vllm.v1.kv_offload.tiering.manager import ( CPUPrimaryTierOffloadingManager, @@ -76,6 +78,50 @@ class TieringOffloadingSpec(CPUOffloadingSpec): cls, extra_config: dict[str, Any] ) -> dict[str, OffloadingMetricMetadata]: metrics = super().build_metric_definitions(extra_config) + metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = ( + OffloadingHistogramMetadata( + documentation=( + "Histogram of total blocking time spent querying secondary " + "tiers for a request, accumulated from first lookup until " + "the request is allocated or finishes, in seconds." + ), + buckets=( + 0.00001, + 0.00005, + 0.0001, + 0.0005, + 0.001, + 0.005, + 0.01, + 0.05, + 0.1, + 0.5, + 1, + ), + ) + ) + metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = ( + OffloadingHistogramMetadata( + documentation=( + "Histogram of wall-clock time from a request's first deferred " + "secondary-tier lookup until the request is allocated or " + "finishes, in seconds." + ), + buckets=( + 0.0001, + 0.0005, + 0.001, + 0.005, + 0.01, + 0.05, + 0.1, + 0.5, + 1, + 5, + 10, + ), + ) + ) secondary_tier_configs = extra_config.get("secondary_tiers", []) if not isinstance(secondary_tier_configs, list): raise ValueError("secondary_tiers must be a list of tier configurations")