Files
e10c10a644 [Core] Add PROMOTION_LATENCY histogram metric for tiering offload (#53910)
Signed-off-by: Wen Zhou <wenzhou@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
2026-10-15 13:47:41 +03:00

424 lines
16 KiB
Python

# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Abstract interfaces and data types for the secondary tiering layer."""
import time
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, ClassVar
import numpy as np
from vllm.v1.kv_offload.base import (
ConfigInfoMapping,
Locality,
LookupResult,
Medium,
OffloadingEvent,
OffloadingMetricMetadata,
OffloadKey,
ReqContext,
RequestOffloadingContext,
ScheduleEndContext,
)
from vllm.v1.kv_offload.tiering.backpressure import (
BackpressureDetector,
)
from vllm.v1.metrics.cache_hit_source import CacheHitSource
if TYPE_CHECKING:
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
OffloadingConnectorStats,
)
from vllm.v1.kv_offload.base import OffloadingSpec
# Type alias for job IDs used in async transfer tracking
JobId = int
TIER_LABEL = "tier"
class TieringOffloadingMetrics:
"""Metric names for TieringOffloadingManager."""
LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds"
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds"
READ_BYTES = "vllm:kv_offload_tiering_read_bytes"
READ_TIME = "vllm:kv_offload_tiering_read_time"
WRITE_BYTES = "vllm:kv_offload_tiering_write_bytes"
WRITE_TIME = "vllm:kv_offload_tiering_write_time"
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
CHUNK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
CHUNK_HITS = "vllm:kv_offload_tiering_chunk_hits"
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
PROMOTION_ALLOCATION_FAILURES = (
"vllm:kv_offload_tiering_promotion_allocation_failures"
)
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
BACKPRESSURE_STORE_LATENCY_EMA = (
"vllm:kv_offload_tiering_backpressure_store_latency_ema"
)
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
PROMOTION_LATENCY = "vllm:kv_offload_tiering_promotion_latency_seconds"
@dataclass
class TransferJob:
"""Metadata for an in-flight async transfer job."""
job_id: JobId
keys: Collection[OffloadKey]
chunk_ids: np.ndarray
is_promotion: bool
req_context: ReqContext
submit_time: float = field(default_factory=time.monotonic)
@dataclass
class JobResult:
"""Result of an async transfer job."""
job_id: JobId
# True if all keys succeeded; False if all or some failed.
success: bool
# Only applicable to promotion jobs. On partial failure, identifies the
# keys that were successfully loaded. None means all keys share the fate
# indicated by `success`. Must be a subset of the job's original keys.
successful_keys: Collection[OffloadKey] | None = None
transfer_time: float | None = None
transfer_bytes: int | None = None
class ParentManager(ABC):
"""Interface for secondary tiers to call back into the tiering manager.
Passed to secondary tiers via serve_external_requests() each step.
The _SecondaryTierFacingParent wrapper implements this, automatically
excluding the calling tier from fan-out operations.
Required call sequence for each remote request:
1. on_new_request(req_context) — set up per-request state
2. lookup(key, req_context) — check chunk availability
(repeat per chunk)
3. create_store_job(keys, req_context) — pin chunks and get a
job handle
4. on_request_finished(req_context) — clean up per-request state
Steps 2-3 may be interleaved. Step 4 must be called even if no
chunks were found, to avoid leaking async lookup state (e.g. in
the fs tier's AsyncLookupManager).
"""
@abstractmethod
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: ...
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: ...
@abstractmethod
def create_store_job(
self,
keys: Collection[OffloadKey],
req_context: ReqContext,
) -> TransferJob: ...
@abstractmethod
def on_request_finished(self, req_context: ReqContext) -> None: ...
class SecondaryTierManager(ABC):
"""Abstract interface for managing a single non-primary offloading tier.
Secondary tiers cannot directly access GPU memory. All data transfers
must go through the CPU (primary) tier:
- Store: GPU → CPU (primary) → secondary (cascade)
- Load: secondary → CPU (primary) → GPU (promotion)
IMPORTANT: All methods run in the Scheduler process and must be
lightweight and non-blocking. submit_load() and submit_store() submit
async jobs; get_finished_jobs() polls for completion.
Methods are called under the tiering manager's lock, so they are never
entered concurrently -- but not always from the same thread. A tier that
sets tolerates_off_thread_polling is also serviced from the manager's
tier-poll thread, and any tier can be reached from there via
ParentManager fan-out. Keep per-thread assumptions out of tier state.
"""
medium: ClassVar[Medium | None] = None
# Cache tier reported for hits served from this tier. Derived from
# ``medium`` unless a subclass sets it; ``None`` medium means unknown.
cache_hit_source: ClassVar[CacheHitSource] = CacheHitSource.EXTERNAL_UNSPECIFIED
def __init_subclass__(cls, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
if "cache_hit_source" not in cls.__dict__ and cls.medium is not None:
cls.cache_hit_source = {
Medium.CPU: CacheHitSource.HOST,
Medium.STORAGE: CacheHitSource.DISK,
}[cls.medium]
# Whether the tiering manager may also service this tier from its
# tier-poll thread, between engine steps, rather than only from
# on_schedule_end(). That thread calls both get_finished_jobs() and
# serve_external_requests(); see poll_tiers(). Set it on a tier that
# answers a counterpart the engine does not drive -- a remote peer, say --
# whose requests would otherwise wait for a step boundary. Overriding
# serve_external_requests() alone does not opt in: setting this asserts
# the tier tolerates those calls off the scheduler thread.
tolerates_off_thread_polling: ClassVar[bool] = False
def __init__(
self,
offloading_spec: "OffloadingSpec",
primary_kv_view: memoryview,
tier_type: str,
backpressure_detector: BackpressureDetector | None = None,
) -> None:
"""Args:
offloading_spec: Offloading configuration.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
tier_type: Tier type identifier, set by SecondaryTierFactory
from the registered tier type.
backpressure_detector: Optional `BackpressureDetector`.
"""
self._offloading_spec = offloading_spec
self._primary_kv_view: memoryview = primary_kv_view
assert primary_kv_view.strides is not None, (
"primary_kv_view.strides cannot be None"
)
self._block_size_bytes: int = primary_kv_view.strides[0]
self.tier_type = tier_type
self.locality: Locality | None = None
self._bp_detector = backpressure_detector
@property
def block_size_bytes(self) -> int:
return self._block_size_bytes
@property
def bp_detector(self) -> BackpressureDetector | None:
return self._bp_detector
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
"""Check whether a chunk exists in this secondary tier.
Args:
key: Offload key to look up.
req_context: per-request context (e.g. kv_transfer_params).
Returns:
HIT if the chunk is present and ready,
MISS if not found,
or RETRY if the chunk is being transferred (retry later).
"""
pass
@abstractmethod
def submit_store(self, job_metadata: TransferJob) -> None:
"""Submit an async job to store chunks from the primary tier to this
secondary tier.
This method must be lightweight and non-blocking: allocate metadata
and submit the transfer, but do NOT perform the data copy on the
calling thread.
Preconditions (guaranteed by the framework):
- ``job_metadata.chunk_ids`` are valid primary-tier slots, pinned
(ref-counted) for the duration of the transfer.
The implementation is responsible for:
1. Filtering out chunks already present in this tier
2. Evicting chunks if capacity is needed
3. Allocating space in this tier
4. Submitting the async transfer (read from primary via chunk_ids)
Report completion via ``get_finished_jobs()``.
Args:
job_metadata: Job metadata including job_id, keys, and chunk_ids
identifying the primary-tier slots to read from.
"""
pass
@abstractmethod
def submit_load(self, job_metadata: TransferJob) -> None:
"""Submit an async job to load chunks from this secondary tier to the
primary tier.
This method must be lightweight and non-blocking: mark chunks as
in-flight and submit the transfer, but do NOT perform the data copy
on the calling thread.
Preconditions (guaranteed by the framework):
- ``job_metadata.chunk_ids`` are allocated primary-tier slots
ready to receive data.
The implementation must copy data from this tier into the
primary-tier slots identified by ``chunk_ids``.
Report completion via ``get_finished_jobs()``.
Args:
job_metadata: Job metadata including job_id, keys, and chunk_ids
identifying the primary-tier slots to write into.
"""
pass
@abstractmethod
def get_finished_jobs(self) -> Iterable[JobResult]:
"""Return all jobs (loads and stores) that completed since the last call.
The framework uses these results to release resources and finalize
transfers.
Returns:
Iterable of JobResult objects for jobs finished since the
last call.
"""
pass
def has_pending_work(self) -> bool:
"""Whether this tier needs the engine to keep stepping.
While True, on_schedule_end() and get_finished_jobs() continue
to be called even when no requests are scheduled.
"""
return False
def take_events(self) -> Iterable[OffloadingEvent]:
"""Take KV events for storage state owned by this tier."""
return ()
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
"""Mark chunks as recently used for eviction policy.
Args:
keys: Offload keys to mark as recently used.
req_context: Per-request context.
"""
return
@abstractmethod
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
"""Called when a new request is first seen by the scheduler.
Returns a RequestOffloadingContext expressing this tier's preference
for how chunks should be offloaded for this request.
Args:
req_context: Per-request context.
"""
pass
def on_request_finished(self, req_context: ReqContext) -> None:
"""Called when a request has finished.
By the time this is called, all per-request calls for this request
(submit_store, submit_load, touch) have already been issued, and none
will follow. Note this does NOT imply the tier's transfers have
completed: jobs already submitted may still be in flight and will
report via get_finished_jobs(). This is the right place to release
per-request bookkeeping.
Args:
req_context: per-request context.
"""
return
def serve_external_requests(self, parent: ParentManager) -> None:
"""Process remotely-originated requests using the parent manager.
Called once per scheduler step, BEFORE _flush_pending_promotions(), and
additionally once per tier-poll round for tiers that set
tolerates_off_thread_polling. The _flush_pending_promotions() ordering
holds only for the per-step call: a promotion this method initiates from
a tier-poll round is submitted at the next on_schedule_end().
The parent handle is valid only for the duration of this call, and the
caller holds the manager lock throughout it -- so a lookup() HIT taken
here stays valid until this call pins it. An implementation must not
release that lock partway through.
Tiers that don't serve external requests leave this as a no-op. A tier
that overrides it without setting tolerates_off_thread_polling is served
once per step only.
"""
return
def on_schedule_end(self, context: ScheduleEndContext) -> None:
"""Called once at the end of each scheduler step.
Args:
context: Per-step context from the scheduler.
"""
return
@abstractmethod
def drain_jobs(self) -> None:
"""Block until every submitted load/store job has completed or failed.
After this returns, no tier I/O is touching the primary memoryview,
and every submitted job's result is available from `get_finished_jobs()`
(yielded by a prior call or queued for the next one). Used by
`TieringOffloadingManager.reset_cache` to release primary slots
without racing with in-flight transfers.
Implementations must not abort a mid-flight transfer: a partial copy
would corrupt either the primary memoryview or the secondary backing
store. Queued (not-yet-started) transfers may be cancelled, but their
failure result must still appear in `get_finished_jobs()`.
"""
pass
def shutdown(self) -> None:
"""Release resources held by this tier (threads, connections, etc.)."""
return
@classmethod
def build_metric_definitions(
cls, extra_config: dict[str, Any]
) -> dict[str, OffloadingMetricMetadata]:
"""Return Prometheus metric definitions emitted by this tier."""
return {}
def get_stats(self) -> "OffloadingConnectorStats | None":
"""Return and reset metric observations collected by this tier."""
return None
@classmethod
def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]:
"""Return the info metric label names of this tier.
TieringOffloadingSpec merges the names of all tiers and adds
TIER_LABEL, so a tier must not declare it. See ConfigInfo.
Args:
tier_config: Configuration dict of this tier.
"""
return ()
def config_info(self) -> ConfigInfoMapping:
"""Return the info metric labels of this tier, for its one series.
TieringOffloadingManager adds TIER_LABEL. See ConfigInfo.
"""
return {}