Signed-off-by: Wen Zhou <wenzhou@redhat.com> Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com> Co-authored-by: OpenAI Codex <noreply@openai.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
424 lines
16 KiB
Python
424 lines
16 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Abstract interfaces and data types for the secondary tiering layer."""
|
|
|
|
import time
|
|
from abc import ABC, abstractmethod
|
|
from collections.abc import Collection, Iterable
|
|
from dataclasses import dataclass, field
|
|
from typing import TYPE_CHECKING, Any, ClassVar
|
|
|
|
import numpy as np
|
|
|
|
from vllm.v1.kv_offload.base import (
|
|
ConfigInfoMapping,
|
|
Locality,
|
|
LookupResult,
|
|
Medium,
|
|
OffloadingEvent,
|
|
OffloadingMetricMetadata,
|
|
OffloadKey,
|
|
ReqContext,
|
|
RequestOffloadingContext,
|
|
ScheduleEndContext,
|
|
)
|
|
from vllm.v1.kv_offload.tiering.backpressure import (
|
|
BackpressureDetector,
|
|
)
|
|
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
|
|
|
if TYPE_CHECKING:
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
|
OffloadingConnectorStats,
|
|
)
|
|
from vllm.v1.kv_offload.base import OffloadingSpec
|
|
|
|
|
|
# Type alias for job IDs used in async transfer tracking
|
|
JobId = int
|
|
|
|
TIER_LABEL = "tier"
|
|
|
|
|
|
class TieringOffloadingMetrics:
|
|
"""Metric names for TieringOffloadingManager."""
|
|
|
|
LOOKUP_SYNC_DELAY = "vllm:kv_offload_tiering_lookup_sync_delay_seconds"
|
|
LOOKUP_ASYNC_DELAY = "vllm:kv_offload_tiering_lookup_async_delay_seconds"
|
|
READ_BYTES = "vllm:kv_offload_tiering_read_bytes"
|
|
READ_TIME = "vllm:kv_offload_tiering_read_time"
|
|
WRITE_BYTES = "vllm:kv_offload_tiering_write_bytes"
|
|
WRITE_TIME = "vllm:kv_offload_tiering_write_time"
|
|
PROMOTION_JOB_FAILURES = "vllm:kv_offload_tiering_promotion_job_failures"
|
|
CASCADE_JOB_FAILURES = "vllm:kv_offload_tiering_cascade_job_failures"
|
|
CHUNK_QUERIES = "vllm:kv_offload_tiering_chunk_queries"
|
|
CHUNK_HITS = "vllm:kv_offload_tiering_chunk_hits"
|
|
PRIMARY_WRITE_USAGE_PERC = "vllm:kv_offload_tiering_primary_write_usage_perc"
|
|
PRIMARY_READ_USAGE_PERC = "vllm:kv_offload_tiering_primary_read_usage_perc"
|
|
PROMOTION_ALLOCATION_FAILURES = (
|
|
"vllm:kv_offload_tiering_promotion_allocation_failures"
|
|
)
|
|
ACTIVE_PROMOTION_JOBS = "vllm:kv_offload_tiering_active_promotion_jobs"
|
|
ACTIVE_CASCADE_JOBS = "vllm:kv_offload_tiering_active_cascade_jobs"
|
|
BACKPRESSURE_STORE_LATENCY_EMA = (
|
|
"vllm:kv_offload_tiering_backpressure_store_latency_ema"
|
|
)
|
|
BACKPRESSURE_STORES_DROPPED = "vllm:kv_offload_tiering_backpressure_stores_dropped"
|
|
BACKPRESSURE_BLOCKS_DROPPED = "vllm:kv_offload_tiering_backpressure_blocks_dropped"
|
|
PROMOTION_LATENCY = "vllm:kv_offload_tiering_promotion_latency_seconds"
|
|
|
|
|
|
@dataclass
|
|
class TransferJob:
|
|
"""Metadata for an in-flight async transfer job."""
|
|
|
|
job_id: JobId
|
|
keys: Collection[OffloadKey]
|
|
chunk_ids: np.ndarray
|
|
is_promotion: bool
|
|
req_context: ReqContext
|
|
submit_time: float = field(default_factory=time.monotonic)
|
|
|
|
|
|
@dataclass
|
|
class JobResult:
|
|
"""Result of an async transfer job."""
|
|
|
|
job_id: JobId
|
|
# True if all keys succeeded; False if all or some failed.
|
|
success: bool
|
|
# Only applicable to promotion jobs. On partial failure, identifies the
|
|
# keys that were successfully loaded. None means all keys share the fate
|
|
# indicated by `success`. Must be a subset of the job's original keys.
|
|
successful_keys: Collection[OffloadKey] | None = None
|
|
transfer_time: float | None = None
|
|
transfer_bytes: int | None = None
|
|
|
|
|
|
class ParentManager(ABC):
|
|
"""Interface for secondary tiers to call back into the tiering manager.
|
|
|
|
Passed to secondary tiers via serve_external_requests() each step.
|
|
The _SecondaryTierFacingParent wrapper implements this, automatically
|
|
excluding the calling tier from fan-out operations.
|
|
|
|
Required call sequence for each remote request:
|
|
1. on_new_request(req_context) — set up per-request state
|
|
2. lookup(key, req_context) — check chunk availability
|
|
(repeat per chunk)
|
|
3. create_store_job(keys, req_context) — pin chunks and get a
|
|
job handle
|
|
4. on_request_finished(req_context) — clean up per-request state
|
|
|
|
Steps 2-3 may be interleaved. Step 4 must be called even if no
|
|
chunks were found, to avoid leaking async lookup state (e.g. in
|
|
the fs tier's AsyncLookupManager).
|
|
"""
|
|
|
|
@abstractmethod
|
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: ...
|
|
|
|
@abstractmethod
|
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult: ...
|
|
|
|
@abstractmethod
|
|
def create_store_job(
|
|
self,
|
|
keys: Collection[OffloadKey],
|
|
req_context: ReqContext,
|
|
) -> TransferJob: ...
|
|
|
|
@abstractmethod
|
|
def on_request_finished(self, req_context: ReqContext) -> None: ...
|
|
|
|
|
|
class SecondaryTierManager(ABC):
|
|
"""Abstract interface for managing a single non-primary offloading tier.
|
|
|
|
Secondary tiers cannot directly access GPU memory. All data transfers
|
|
must go through the CPU (primary) tier:
|
|
- Store: GPU → CPU (primary) → secondary (cascade)
|
|
- Load: secondary → CPU (primary) → GPU (promotion)
|
|
|
|
IMPORTANT: All methods run in the Scheduler process and must be
|
|
lightweight and non-blocking. submit_load() and submit_store() submit
|
|
async jobs; get_finished_jobs() polls for completion.
|
|
|
|
Methods are called under the tiering manager's lock, so they are never
|
|
entered concurrently -- but not always from the same thread. A tier that
|
|
sets tolerates_off_thread_polling is also serviced from the manager's
|
|
tier-poll thread, and any tier can be reached from there via
|
|
ParentManager fan-out. Keep per-thread assumptions out of tier state.
|
|
"""
|
|
|
|
medium: ClassVar[Medium | None] = None
|
|
# Cache tier reported for hits served from this tier. Derived from
|
|
# ``medium`` unless a subclass sets it; ``None`` medium means unknown.
|
|
cache_hit_source: ClassVar[CacheHitSource] = CacheHitSource.EXTERNAL_UNSPECIFIED
|
|
|
|
def __init_subclass__(cls, **kwargs: Any) -> None:
|
|
super().__init_subclass__(**kwargs)
|
|
if "cache_hit_source" not in cls.__dict__ and cls.medium is not None:
|
|
cls.cache_hit_source = {
|
|
Medium.CPU: CacheHitSource.HOST,
|
|
Medium.STORAGE: CacheHitSource.DISK,
|
|
}[cls.medium]
|
|
|
|
# Whether the tiering manager may also service this tier from its
|
|
# tier-poll thread, between engine steps, rather than only from
|
|
# on_schedule_end(). That thread calls both get_finished_jobs() and
|
|
# serve_external_requests(); see poll_tiers(). Set it on a tier that
|
|
# answers a counterpart the engine does not drive -- a remote peer, say --
|
|
# whose requests would otherwise wait for a step boundary. Overriding
|
|
# serve_external_requests() alone does not opt in: setting this asserts
|
|
# the tier tolerates those calls off the scheduler thread.
|
|
tolerates_off_thread_polling: ClassVar[bool] = False
|
|
|
|
def __init__(
|
|
self,
|
|
offloading_spec: "OffloadingSpec",
|
|
primary_kv_view: memoryview,
|
|
tier_type: str,
|
|
backpressure_detector: BackpressureDetector | None = None,
|
|
) -> None:
|
|
"""Args:
|
|
offloading_spec: Offloading configuration.
|
|
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
|
tier_type: Tier type identifier, set by SecondaryTierFactory
|
|
from the registered tier type.
|
|
backpressure_detector: Optional `BackpressureDetector`.
|
|
|
|
"""
|
|
self._offloading_spec = offloading_spec
|
|
self._primary_kv_view: memoryview = primary_kv_view
|
|
assert primary_kv_view.strides is not None, (
|
|
"primary_kv_view.strides cannot be None"
|
|
)
|
|
self._block_size_bytes: int = primary_kv_view.strides[0]
|
|
self.tier_type = tier_type
|
|
self.locality: Locality | None = None
|
|
self._bp_detector = backpressure_detector
|
|
|
|
@property
|
|
def block_size_bytes(self) -> int:
|
|
return self._block_size_bytes
|
|
|
|
@property
|
|
def bp_detector(self) -> BackpressureDetector | None:
|
|
return self._bp_detector
|
|
|
|
@abstractmethod
|
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
|
"""Check whether a chunk exists in this secondary tier.
|
|
|
|
Args:
|
|
key: Offload key to look up.
|
|
req_context: per-request context (e.g. kv_transfer_params).
|
|
|
|
Returns:
|
|
HIT if the chunk is present and ready,
|
|
MISS if not found,
|
|
or RETRY if the chunk is being transferred (retry later).
|
|
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
|
"""Submit an async job to store chunks from the primary tier to this
|
|
secondary tier.
|
|
|
|
This method must be lightweight and non-blocking: allocate metadata
|
|
and submit the transfer, but do NOT perform the data copy on the
|
|
calling thread.
|
|
|
|
Preconditions (guaranteed by the framework):
|
|
- ``job_metadata.chunk_ids`` are valid primary-tier slots, pinned
|
|
(ref-counted) for the duration of the transfer.
|
|
|
|
The implementation is responsible for:
|
|
1. Filtering out chunks already present in this tier
|
|
2. Evicting chunks if capacity is needed
|
|
3. Allocating space in this tier
|
|
4. Submitting the async transfer (read from primary via chunk_ids)
|
|
|
|
Report completion via ``get_finished_jobs()``.
|
|
|
|
Args:
|
|
job_metadata: Job metadata including job_id, keys, and chunk_ids
|
|
identifying the primary-tier slots to read from.
|
|
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
|
"""Submit an async job to load chunks from this secondary tier to the
|
|
primary tier.
|
|
|
|
This method must be lightweight and non-blocking: mark chunks as
|
|
in-flight and submit the transfer, but do NOT perform the data copy
|
|
on the calling thread.
|
|
|
|
Preconditions (guaranteed by the framework):
|
|
- ``job_metadata.chunk_ids`` are allocated primary-tier slots
|
|
ready to receive data.
|
|
|
|
The implementation must copy data from this tier into the
|
|
primary-tier slots identified by ``chunk_ids``.
|
|
|
|
Report completion via ``get_finished_jobs()``.
|
|
|
|
Args:
|
|
job_metadata: Job metadata including job_id, keys, and chunk_ids
|
|
identifying the primary-tier slots to write into.
|
|
|
|
"""
|
|
pass
|
|
|
|
@abstractmethod
|
|
def get_finished_jobs(self) -> Iterable[JobResult]:
|
|
"""Return all jobs (loads and stores) that completed since the last call.
|
|
|
|
The framework uses these results to release resources and finalize
|
|
transfers.
|
|
|
|
Returns:
|
|
Iterable of JobResult objects for jobs finished since the
|
|
last call.
|
|
|
|
"""
|
|
pass
|
|
|
|
def has_pending_work(self) -> bool:
|
|
"""Whether this tier needs the engine to keep stepping.
|
|
|
|
While True, on_schedule_end() and get_finished_jobs() continue
|
|
to be called even when no requests are scheduled.
|
|
"""
|
|
return False
|
|
|
|
def take_events(self) -> Iterable[OffloadingEvent]:
|
|
"""Take KV events for storage state owned by this tier."""
|
|
return ()
|
|
|
|
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
|
"""Mark chunks as recently used for eviction policy.
|
|
|
|
Args:
|
|
keys: Offload keys to mark as recently used.
|
|
req_context: Per-request context.
|
|
|
|
"""
|
|
return
|
|
|
|
@abstractmethod
|
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
|
"""Called when a new request is first seen by the scheduler.
|
|
|
|
Returns a RequestOffloadingContext expressing this tier's preference
|
|
for how chunks should be offloaded for this request.
|
|
|
|
Args:
|
|
req_context: Per-request context.
|
|
|
|
"""
|
|
pass
|
|
|
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
|
"""Called when a request has finished.
|
|
|
|
By the time this is called, all per-request calls for this request
|
|
(submit_store, submit_load, touch) have already been issued, and none
|
|
will follow. Note this does NOT imply the tier's transfers have
|
|
completed: jobs already submitted may still be in flight and will
|
|
report via get_finished_jobs(). This is the right place to release
|
|
per-request bookkeeping.
|
|
|
|
Args:
|
|
req_context: per-request context.
|
|
|
|
"""
|
|
return
|
|
|
|
def serve_external_requests(self, parent: ParentManager) -> None:
|
|
"""Process remotely-originated requests using the parent manager.
|
|
|
|
Called once per scheduler step, BEFORE _flush_pending_promotions(), and
|
|
additionally once per tier-poll round for tiers that set
|
|
tolerates_off_thread_polling. The _flush_pending_promotions() ordering
|
|
holds only for the per-step call: a promotion this method initiates from
|
|
a tier-poll round is submitted at the next on_schedule_end().
|
|
|
|
The parent handle is valid only for the duration of this call, and the
|
|
caller holds the manager lock throughout it -- so a lookup() HIT taken
|
|
here stays valid until this call pins it. An implementation must not
|
|
release that lock partway through.
|
|
|
|
Tiers that don't serve external requests leave this as a no-op. A tier
|
|
that overrides it without setting tolerates_off_thread_polling is served
|
|
once per step only.
|
|
"""
|
|
return
|
|
|
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
|
"""Called once at the end of each scheduler step.
|
|
|
|
Args:
|
|
context: Per-step context from the scheduler.
|
|
|
|
"""
|
|
return
|
|
|
|
@abstractmethod
|
|
def drain_jobs(self) -> None:
|
|
"""Block until every submitted load/store job has completed or failed.
|
|
|
|
After this returns, no tier I/O is touching the primary memoryview,
|
|
and every submitted job's result is available from `get_finished_jobs()`
|
|
(yielded by a prior call or queued for the next one). Used by
|
|
`TieringOffloadingManager.reset_cache` to release primary slots
|
|
without racing with in-flight transfers.
|
|
|
|
Implementations must not abort a mid-flight transfer: a partial copy
|
|
would corrupt either the primary memoryview or the secondary backing
|
|
store. Queued (not-yet-started) transfers may be cancelled, but their
|
|
failure result must still appear in `get_finished_jobs()`.
|
|
"""
|
|
pass
|
|
|
|
def shutdown(self) -> None:
|
|
"""Release resources held by this tier (threads, connections, etc.)."""
|
|
return
|
|
|
|
@classmethod
|
|
def build_metric_definitions(
|
|
cls, extra_config: dict[str, Any]
|
|
) -> dict[str, OffloadingMetricMetadata]:
|
|
"""Return Prometheus metric definitions emitted by this tier."""
|
|
return {}
|
|
|
|
def get_stats(self) -> "OffloadingConnectorStats | None":
|
|
"""Return and reset metric observations collected by this tier."""
|
|
return None
|
|
|
|
@classmethod
|
|
def config_info_keys(cls, tier_config: dict[str, Any]) -> tuple[str, ...]:
|
|
"""Return the info metric label names of this tier.
|
|
|
|
TieringOffloadingSpec merges the names of all tiers and adds
|
|
TIER_LABEL, so a tier must not declare it. See ConfigInfo.
|
|
|
|
Args:
|
|
tier_config: Configuration dict of this tier.
|
|
|
|
"""
|
|
return ()
|
|
|
|
def config_info(self) -> ConfigInfoMapping:
|
|
"""Return the info metric labels of this tier, for its one series.
|
|
|
|
TieringOffloadingManager adds TIER_LABEL. See ConfigInfo.
|
|
"""
|
|
return {}
|