[KV Offload] Split tiering_lookup_delay into sync/async histograms (#47679)
Signed-off-by: srinivas_oo7 <sklinkedin0120@gmail.com> Co-authored-by: srinivas_oo7 <sklinkedin0120@gmail.com>
This commit is contained in:
co-authored by
srinivas_oo7
parent
7d56fe2adc
commit
ce65385618
@@ -35,6 +35,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
@@ -233,9 +234,9 @@ class TestTieringOffloadingManager:
|
||||
secondary_tiers=[self.secondary_tier1, self.secondary_tier2],
|
||||
)
|
||||
|
||||
def _simulate_on_schedule_end(self):
|
||||
def _simulate_on_schedule_end(self, new_req_ids: list[str] | None = None):
|
||||
"""Simulate end of scheduler step: lifecycle flush + drain events."""
|
||||
ctx = ScheduleEndContext(new_req_ids=[], preempted_req_ids=())
|
||||
ctx = ScheduleEndContext(new_req_ids=new_req_ids or [], preempted_req_ids=())
|
||||
self.manager.on_schedule_end(ctx)
|
||||
list(self.manager.take_events())
|
||||
|
||||
@@ -395,6 +396,87 @@ class TestTieringOffloadingManager:
|
||||
# Next lookup should succeed
|
||||
assert count_hits(self.manager, blocks) == 3
|
||||
|
||||
def test_lookup_reports_sync_delay_for_resolved_lookups(self, manager_setup):
|
||||
"""Resolved lookups report one sync delay sample on allocation."""
|
||||
self._start_request()
|
||||
blocks = to_keys(range(2))
|
||||
|
||||
# No tier has these blocks: they resolve immediately as misses.
|
||||
for block in blocks:
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.MISS
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
)
|
||||
|
||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in reduced
|
||||
|
||||
def test_lookup_reports_async_delay_across_promotion(self, manager_setup):
|
||||
"""A new request reports async delay at schedule end."""
|
||||
self._start_request()
|
||||
block = to_keys(range(1))[0]
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
|
||||
# First lookup finds the block in a secondary tier and defers.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.RETRY
|
||||
|
||||
# The first scheduler step reports async delay for the new request.
|
||||
self._simulate_on_schedule_end(new_req_ids=[_CTX.req_id])
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
|
||||
|
||||
# Promotion completes on the next scheduler step.
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Next lookup resolves via the now-promoted primary-tier block.
|
||||
assert self.manager.lookup(block, _CTX) is LookupResult.HIT
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
)
|
||||
|
||||
def test_lookup_reports_async_delay_on_request_finish(self, manager_setup):
|
||||
"""Never-allocated lookup delays flush at teardown."""
|
||||
ctx = ReqContext(req_id="req_lookup_finish")
|
||||
self._start_request(ctx)
|
||||
block = to_keys(range(1))[0]
|
||||
self.secondary_tier1.blocks[block] = True
|
||||
|
||||
# Lookup finds the block in a secondary tier and defers.
|
||||
assert self.manager.lookup(block, ctx) is LookupResult.RETRY
|
||||
|
||||
self._simulate_on_schedule_end()
|
||||
stats = self.manager.get_stats()
|
||||
if stats is not None:
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
)
|
||||
assert f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count" not in (
|
||||
stats.reduce()
|
||||
)
|
||||
|
||||
# Request finishes before the deferred lookup is ever resolved.
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
stats = self.manager.get_stats()
|
||||
assert stats is not None
|
||||
reduced = stats.reduce()
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_SYNC_DELAY}_count"] == 1
|
||||
assert reduced[f"{TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY}_count"] == 1
|
||||
|
||||
def test_partial_lookup(self, manager_setup):
|
||||
"""Test lookup with partial hits."""
|
||||
blocks = to_keys(range(5))
|
||||
|
||||
@@ -31,6 +31,13 @@ if TYPE_CHECKING:
|
||||
JobId = int
|
||||
|
||||
|
||||
class TieringOffloadingMetrics:
|
||||
"""Metric names for TieringOffloadingManager."""
|
||||
|
||||
LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds"
|
||||
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds"
|
||||
|
||||
|
||||
@dataclass
|
||||
class JobMetadata:
|
||||
"""Metadata for an in-flight async transfer job."""
|
||||
|
||||
@@ -20,6 +20,7 @@ Key Design Principles:
|
||||
protecting blocks from eviction until complete_read() is called
|
||||
"""
|
||||
|
||||
import time
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
@@ -50,6 +51,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
ParentManager,
|
||||
SecondaryTierManager,
|
||||
TieringOffloadingMetrics,
|
||||
)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -70,6 +72,10 @@ class RequestState:
|
||||
pending_primary_stores: int = 0
|
||||
is_finished: bool = False
|
||||
request_level_tiers: set[SecondaryTierManager] | None = None
|
||||
sync_lookup_delay: float = 0.0
|
||||
# time.monotonic() of this request's first deferred secondary-tier lookup;
|
||||
# None once consumed (observed) or while no secondary lookup is pending.
|
||||
secondary_lookup_start_time: float | None = None
|
||||
|
||||
|
||||
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
@@ -212,6 +218,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
for tier in self.secondary_tiers
|
||||
}
|
||||
|
||||
# Buffers manager-level observations (e.g. lookup delay) between
|
||||
# get_stats() calls; merged in and reset each time get_stats() runs.
|
||||
self._stats = OffloadingConnectorStats()
|
||||
|
||||
def _next_job_id(self) -> JobId:
|
||||
"""Generate a unique job ID for async transfer tracking."""
|
||||
job_id = self._job_id_counter
|
||||
@@ -301,28 +311,68 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# in time for a promotion this lookup may initiate.
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
req_state = self._req_state.get(req_context.req_id)
|
||||
|
||||
primary_hit = self.primary_tier.lookup(key, req_context)
|
||||
if primary_hit is LookupResult.HIT:
|
||||
return LookupResult.HIT
|
||||
if primary_hit is LookupResult.HIT_PENDING:
|
||||
return LookupResult.HIT_PENDING
|
||||
|
||||
lookup_start = time.monotonic()
|
||||
any_retry = False
|
||||
for tier in self.secondary_tiers:
|
||||
if tier is exclude_tier:
|
||||
continue
|
||||
result = tier.lookup(key, req_context)
|
||||
if result is LookupResult.HIT:
|
||||
if not self._initiate_promotion(tier, key, req_context):
|
||||
return LookupResult.MISS
|
||||
return LookupResult.RETRY
|
||||
promoted = self._initiate_promotion(tier, key, req_context)
|
||||
self._accumulate_lookup_sync_delay(req_state, lookup_start)
|
||||
if (
|
||||
req_state is not None
|
||||
and promoted
|
||||
and req_state.secondary_lookup_start_time is None
|
||||
):
|
||||
req_state.secondary_lookup_start_time = lookup_start
|
||||
return LookupResult.MISS if not promoted else LookupResult.RETRY
|
||||
if result is LookupResult.RETRY:
|
||||
any_retry = True
|
||||
|
||||
self._accumulate_lookup_sync_delay(req_state, lookup_start)
|
||||
if any_retry:
|
||||
if req_state is not None and req_state.secondary_lookup_start_time is None:
|
||||
req_state.secondary_lookup_start_time = lookup_start
|
||||
return LookupResult.RETRY
|
||||
return LookupResult.MISS
|
||||
|
||||
def _accumulate_lookup_sync_delay(
|
||||
self, req_state: RequestState | None, start_time: float
|
||||
) -> None:
|
||||
"""Accumulate secondary-tier lookup time until allocation or finish."""
|
||||
if req_state is not None:
|
||||
req_state.sync_lookup_delay += time.monotonic() - start_time
|
||||
|
||||
def _maybe_observe_lookup_sync_delay(self, req_state: RequestState) -> None:
|
||||
delay = req_state.sync_lookup_delay
|
||||
if delay == 0:
|
||||
return
|
||||
req_state.sync_lookup_delay = 0.0
|
||||
self._stats.observe_histogram(
|
||||
TieringOffloadingMetrics.LOOKUP_SYNC_DELAY,
|
||||
delay,
|
||||
)
|
||||
|
||||
def _maybe_observe_lookup_async_delay(self, req_state: RequestState) -> None:
|
||||
"""Flush a pending deferred secondary-tier lookup timer, if any."""
|
||||
start_time = req_state.secondary_lookup_start_time
|
||||
if start_time is None:
|
||||
return
|
||||
req_state.secondary_lookup_start_time = None
|
||||
self._stats.observe_histogram(
|
||||
TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY,
|
||||
time.monotonic() - start_time,
|
||||
)
|
||||
|
||||
def _initiate_promotion(
|
||||
self,
|
||||
tier: SecondaryTierManager,
|
||||
@@ -666,6 +716,8 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
if tier is exclude_tier:
|
||||
continue
|
||||
tier.on_request_finished(state.req_context)
|
||||
self._maybe_observe_lookup_sync_delay(state)
|
||||
self._maybe_observe_lookup_async_delay(state)
|
||||
del self._req_state[req_id]
|
||||
|
||||
@override
|
||||
@@ -692,6 +744,13 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_schedule_end(context)
|
||||
|
||||
for req_id in context.new_req_ids:
|
||||
state = self._req_state.get(req_id)
|
||||
if state is None:
|
||||
continue
|
||||
self._maybe_observe_lookup_sync_delay(state)
|
||||
self._maybe_observe_lookup_async_delay(state)
|
||||
|
||||
@override
|
||||
def has_pending_work(self) -> bool:
|
||||
# In-flight primary<->secondary transfers (pending promotions are
|
||||
@@ -745,6 +804,8 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
continue
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_request_finished(state.req_context)
|
||||
self._maybe_observe_lookup_sync_delay(state)
|
||||
self._maybe_observe_lookup_async_delay(state)
|
||||
finished_req_ids.append(req_id)
|
||||
|
||||
self.primary_tier.reset_cache()
|
||||
@@ -769,6 +830,13 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
else:
|
||||
stats.aggregate(tier_stats)
|
||||
|
||||
if not self._stats.is_empty():
|
||||
if stats is None:
|
||||
stats = self._stats
|
||||
else:
|
||||
stats.aggregate(self._stats)
|
||||
self._stats = OffloadingConnectorStats()
|
||||
|
||||
return stats
|
||||
|
||||
@override
|
||||
|
||||
@@ -39,6 +39,7 @@ from typing_extensions import override
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
CanonicalKVCaches,
|
||||
OffloadingHistogramMetadata,
|
||||
OffloadingManager,
|
||||
OffloadingMetricMetadata,
|
||||
)
|
||||
@@ -46,6 +47,7 @@ from vllm.v1.kv_offload.config import OffloadingConfig
|
||||
from vllm.v1.kv_offload.cpu.gpu_worker import CPUOffloadingWorker
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.spec import CPUOffloadingSpec
|
||||
from vllm.v1.kv_offload.tiering.base import TieringOffloadingMetrics
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
@@ -76,6 +78,50 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
cls, extra_config: dict[str, Any]
|
||||
) -> dict[str, OffloadingMetricMetadata]:
|
||||
metrics = super().build_metric_definitions(extra_config)
|
||||
metrics[TieringOffloadingMetrics.LOOKUP_SYNC_DELAY] = (
|
||||
OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of total blocking time spent querying secondary "
|
||||
"tiers for a request, accumulated from first lookup until "
|
||||
"the request is allocated or finishes, in seconds."
|
||||
),
|
||||
buckets=(
|
||||
0.00001,
|
||||
0.00005,
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
),
|
||||
)
|
||||
)
|
||||
metrics[TieringOffloadingMetrics.LOOKUP_ASYNC_DELAY] = (
|
||||
OffloadingHistogramMetadata(
|
||||
documentation=(
|
||||
"Histogram of wall-clock time from a request's first deferred "
|
||||
"secondary-tier lookup until the request is allocated or "
|
||||
"finishes, in seconds."
|
||||
),
|
||||
buckets=(
|
||||
0.0001,
|
||||
0.0005,
|
||||
0.001,
|
||||
0.005,
|
||||
0.01,
|
||||
0.05,
|
||||
0.1,
|
||||
0.5,
|
||||
1,
|
||||
5,
|
||||
10,
|
||||
),
|
||||
)
|
||||
)
|
||||
secondary_tier_configs = extra_config.get("secondary_tiers", [])
|
||||
if not isinstance(secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
Reference in New Issue
Block a user