Signed-off-by: Liran Schour <lirans@il.ibm.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
1283 lines
51 KiB
Python
1283 lines
51 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""TieringOffloadingManager: Multi-tier KV cache offloading orchestrator.
|
|
|
|
This manager coordinates between a CPU primary tier (with direct GPU access)
|
|
and zero or more secondary tiers (Storage, Network, etc.) to provide
|
|
hierarchical KV cache offloading.
|
|
|
|
Key Design Principles:
|
|
1. Always offload to all tiers — When a chunk is stored to the primary tier,
|
|
it is cascaded to ALL secondary tiers
|
|
2. Primary tier is the gateway — Secondary tiers cannot access GPU memory
|
|
directly; all data flows through the CPU primary tier
|
|
3. Staged promotion — Chunks in secondary tiers must be promoted to the
|
|
primary tier before GPU can access them
|
|
4. Transparent retry mechanism — Return None from lookup() to signal
|
|
"data is being promoted, try later"
|
|
5. ref_cnt as eviction protection — primary.prepare_read() increments ref_cnt,
|
|
protecting chunks from eviction until complete_read() is called
|
|
"""
|
|
|
|
import threading
|
|
import time
|
|
from collections.abc import Collection, Iterable, Sequence
|
|
from dataclasses import dataclass, field
|
|
from typing import NamedTuple
|
|
|
|
import numpy as np
|
|
from typing_extensions import override
|
|
|
|
from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
|
OffloadingConnectorStats,
|
|
)
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.kv_offload.base import (
|
|
ConfigInfoMapping,
|
|
LoadStoreSpec,
|
|
LookupResult,
|
|
OffloadingEvent,
|
|
OffloadingManager,
|
|
OffloadKey,
|
|
OffloadPolicy,
|
|
PrepareStoreOutput,
|
|
ReqContext,
|
|
RequestOffloadingContext,
|
|
ScheduleEndContext,
|
|
)
|
|
from vllm.v1.kv_offload.cpu.common import CPULoadStoreSpec
|
|
from vllm.v1.kv_offload.cpu.manager import CPUOffloadingManager
|
|
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
|
from vllm.v1.kv_offload.tiering.base import (
|
|
TIER_LABEL,
|
|
JobId,
|
|
JobResult,
|
|
ParentManager,
|
|
SecondaryTierManager,
|
|
TransferJob,
|
|
)
|
|
from vllm.v1.kv_offload.tiering.metrics import TieringMetricsTracker
|
|
from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
# Default pause between tier-poll rounds; the user-facing default of the
|
|
# ``tier_poll_interval_s`` extra-config key. A round is a non-blocking sweep, so
|
|
# this sets how long a peer's request can sit unserved; 1 ms is three orders of
|
|
# magnitude below the model step it hides, and matches the pause the p2p tier's
|
|
# own drain loops and EngineCoreProc's GIL-yield sleep already use.
|
|
DEFAULT_TIER_POLL_INTERVAL_S = 0.001
|
|
# How long a round waits for the manager lock before giving up and retrying.
|
|
# Bounded so the thread stays responsive to shutdown even if a caller holds the
|
|
# lock for a long time, or leaked it.
|
|
_TIER_POLL_LOCK_TIMEOUT_S = 1.0
|
|
# Warn after this much continuous failure to acquire the lock. Above the 5 s
|
|
# warning drain_jobs() emits, so a slow reset_cache() does not warn twice.
|
|
_TIER_POLL_STALL_WARN_S = 30.0
|
|
# Give up after rounds have failed continuously for this long. Counted in time,
|
|
# not rounds: at the default 1 ms interval a round budget is spent in a blink,
|
|
# so a transient fault (a peer reconnecting) would turn the thread off for
|
|
# good. on_schedule_end() still serves the tiers, so the fallback is per-step
|
|
# servicing rather than none.
|
|
_TIER_POLL_GIVE_UP_AFTER_S = 30.0
|
|
# Minimum spacing between logged tracebacks for a run of failing rounds, which
|
|
# would otherwise log one per interval.
|
|
_TIER_POLL_ERROR_LOG_INTERVAL_S = 1.0
|
|
# How long shutdown() waits for the thread to finish its round.
|
|
_TIER_POLL_JOIN_TIMEOUT_S = 5.0
|
|
|
|
|
|
@dataclass
|
|
class PendingPromotion:
|
|
"""Accumulator for chunks awaiting submit_load() for one (tier, request)."""
|
|
|
|
req_context: ReqContext
|
|
keys: list[OffloadKey] = field(default_factory=list)
|
|
chunk_ids: list[int] = field(default_factory=list)
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class RequestState:
|
|
req_context: ReqContext
|
|
pending_primary_stores: int = 0
|
|
is_finished: bool = False
|
|
request_level_tiers: set[int] | None = None
|
|
pending_cascade_keys: list[OffloadKey] = field(default_factory=list)
|
|
|
|
|
|
class JobMetadata(NamedTuple):
|
|
transfer_job: TransferJob
|
|
tier_idx: int
|
|
|
|
|
|
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
|
"""CPUOffloadingManager with a primary/secondary transfer interface.
|
|
|
|
The inherited prepare_store/complete_store/prepare_load/complete_load are the
|
|
GPU-facing OffloadingManager interface. These aliases expose the same operations
|
|
from the secondary tier perspective, where read/write refers to secondary
|
|
accessing primary. This avoids confusion when reading TieringOffloadingManager
|
|
code (e.g. calling prepare_load inside a cascade/store path would be misleading).
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
num_chunks: int,
|
|
mmap_region: SharedOffloadRegion,
|
|
cache_policy: str = "lru",
|
|
cache_policy_module_path: str | None = None,
|
|
enable_events: bool = False,
|
|
):
|
|
super().__init__(
|
|
num_chunks=num_chunks,
|
|
cache_policy=cache_policy,
|
|
cache_policy_module_path=cache_policy_module_path,
|
|
enable_events=enable_events,
|
|
)
|
|
self._mmap_region = mmap_region
|
|
# read/write is for CPU<->secondary transfers,
|
|
# load/store is for CPU<->GPU transfers.
|
|
# These aliases avoid calling prepare_load inside a store path.
|
|
self.complete_read = self.complete_load
|
|
self.prepare_write = self.prepare_store
|
|
self.complete_write = self.complete_store
|
|
|
|
self._kv_memoryview = mmap_region.create_kv_memoryview()
|
|
|
|
def prepare_read(
|
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
|
) -> LoadStoreSpec:
|
|
"""Pin chunks for a CPU-to-secondary transfer.
|
|
|
|
Cascade reads are implementation details of tiering, not additional
|
|
request accesses, so they must not alter request-scoped recency.
|
|
"""
|
|
return self._prepare_load(keys, req_context, record_access=False)
|
|
|
|
def get_kv_memoryview(self) -> memoryview:
|
|
"""Return the memoryview over the primary tier's KV cache buffer.
|
|
|
|
The view has shape (num_chunks, row_stride_bytes) and is backed by the
|
|
SharedOffloadRegion mmap. Secondary tiers address chunk *c* as
|
|
``view[c]``.
|
|
"""
|
|
return self._kv_memoryview
|
|
|
|
@override
|
|
def shutdown(self) -> None:
|
|
super().shutdown()
|
|
self._kv_memoryview.release()
|
|
self._mmap_region.cleanup()
|
|
|
|
|
|
class _SecondaryTierFacingParent(ParentManager):
|
|
"""Wrapper that implements ParentManager by delegating to the
|
|
TieringOffloadingManager with exclude_tier_idx set to the origin tier."""
|
|
|
|
__slots__ = ("_m", "_origin_idx")
|
|
|
|
def __init__(
|
|
self,
|
|
manager: "TieringOffloadingManager",
|
|
tier_idx: int,
|
|
):
|
|
self._m = manager
|
|
self._origin_idx = tier_idx
|
|
|
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
|
return self._m.on_new_request(req_context, exclude_tier_idx=self._origin_idx)
|
|
|
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
|
return self._m.lookup(key, req_context, exclude_tier_idx=self._origin_idx)
|
|
|
|
def create_store_job(
|
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
|
) -> TransferJob:
|
|
return self._m.create_store_job(keys, req_context, self._origin_idx)
|
|
|
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
|
return self._m.on_request_finished(
|
|
req_context, exclude_tier_idx=self._origin_idx
|
|
)
|
|
|
|
|
|
class TieringOffloadingManager(OffloadingManager):
|
|
"""Orchestrates multi-tier KV cache offloading.
|
|
|
|
This manager coordinates between a CPU primary tier (with direct GPU access)
|
|
and zero or more secondary tiers (Storage, Network, etc.) to provide
|
|
hierarchical KV cache offloading.
|
|
|
|
Key internal state:
|
|
- Minimal state tracking; relies on secondary tiers to report completion
|
|
via get_finished_jobs()
|
|
- Secondary tiers return JobResult objects containing all necessary
|
|
information
|
|
- job_id_counter: monotonically increasing counter for job IDs
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
primary_tier: CPUPrimaryTierOffloadingManager,
|
|
secondary_tiers: list[SecondaryTierManager] | None = None,
|
|
tier_poll_interval_s: float = DEFAULT_TIER_POLL_INTERVAL_S,
|
|
):
|
|
"""Initialize the TieringOffloadingManager.
|
|
|
|
Args:
|
|
primary_tier: The primary tier manager (CPU-based).
|
|
secondary_tiers: List of secondary tier managers (e.g., Storage,
|
|
Network). Can be None or empty list.
|
|
tier_poll_interval_s: Pause between tier-poll rounds for
|
|
tiers that set tolerates_off_thread_polling. Zero or negative
|
|
disables the thread, leaving those tiers serviced once per
|
|
engine step as before.
|
|
|
|
"""
|
|
# Guards this manager and every tier it composes, the primary tier
|
|
# included. A real lock because the tier-poll thread enters the
|
|
# manager too; the primary tier has none of its own.
|
|
self._lock = threading.Lock()
|
|
self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier
|
|
self.secondary_tiers = secondary_tiers or []
|
|
|
|
self._job_id_counter: int = 0
|
|
# Job tracking: maps job_id to metadata for all in-flight transfers.
|
|
# TransferJob.is_promotion distinguishes direction:
|
|
# True: secondary → primary (promotion)
|
|
# False: primary → secondary (cascade)
|
|
self._jobs: dict[JobId, JobMetadata] = {}
|
|
primary_view = self.primary_tier.get_kv_memoryview()
|
|
assert primary_view.strides is not None
|
|
self._metrics = TieringMetricsTracker(
|
|
tier_types=[tier.tier_type for tier in self.secondary_tiers],
|
|
num_primary_chunks=self.primary_tier._num_chunks,
|
|
primary_chunk_size=primary_view.strides[0],
|
|
)
|
|
|
|
# Pending promotion requests accumulated during lookup() calls; flushed
|
|
# as one batched submit_load() per (tier, request) in on_schedule_end().
|
|
# Outer key: tier index. Inner key: req_context.req_id — the same ReqContext
|
|
# object is reused for all chunk lookups of a given request per engine step.
|
|
self._pending_load_submissions: dict[int, dict[str, PendingPromotion]] = {}
|
|
|
|
# Gate for once-per-step execution of _maybe_process_finished_jobs().
|
|
# Reset at the end of each step in on_schedule_end().
|
|
self._processed_jobs_this_step: bool = False
|
|
|
|
# Per-request state for prepared GPU->primary stores and finalization.
|
|
# Secondary tiers are finalized only after pending primary stores reach
|
|
# complete_store(), since complete_store() can still submit cascades.
|
|
self._req_state: dict[str, RequestState] = {}
|
|
|
|
# Preserve the original tier for this request's cache-hit metrics,
|
|
# even after its KV blocks are promoted into host memory
|
|
# (otherwise a disk or P2P hit would be mislabeled as a host hit).
|
|
self._request_load_sources: dict[str, dict[OffloadKey, CacheHitSource]] = {}
|
|
|
|
# Cached ParentManager wrappers for each secondary tier.
|
|
self._tier_parents: dict[SecondaryTierManager, _SecondaryTierFacingParent] = {
|
|
tier: _SecondaryTierFacingParent(self, tier_idx)
|
|
for tier_idx, tier in enumerate(self.secondary_tiers)
|
|
}
|
|
|
|
self._tier_index: dict[SecondaryTierManager, int] = {
|
|
tier: i for i, tier in enumerate(self.secondary_tiers)
|
|
}
|
|
|
|
# Tiers that serve external requests, and so must also be serviced
|
|
# between engine steps.
|
|
self._polled_tiers: list[SecondaryTierManager] = [
|
|
tier for tier in self.secondary_tiers if tier.tolerates_off_thread_polling
|
|
]
|
|
self._tier_poll_interval_s = tier_poll_interval_s
|
|
self._tier_poll_stop = threading.Event()
|
|
self._tier_poll_thread: threading.Thread | None = None
|
|
self._start_tier_poll_thread()
|
|
|
|
@property
|
|
@override
|
|
def lock(self) -> threading.Lock:
|
|
return self._lock
|
|
|
|
# ------------------------------------------------------------------
|
|
# Tier polling
|
|
# ------------------------------------------------------------------
|
|
|
|
def _start_tier_poll_thread(self) -> None:
|
|
"""Start the tier-poll thread, if any tier asked for one."""
|
|
if not self._polled_tiers or self._tier_poll_interval_s <= 0:
|
|
return
|
|
self._tier_poll_thread = threading.Thread(
|
|
target=self._tier_poll_loop,
|
|
name="vllm_offload_tier_poll",
|
|
daemon=True,
|
|
)
|
|
self._tier_poll_thread.start()
|
|
logger.info(
|
|
"KV offload tier-poll thread started for tier(s) %s, interval %.3fs",
|
|
[tier.tier_type for tier in self._polled_tiers],
|
|
self._tier_poll_interval_s,
|
|
)
|
|
|
|
def _stop_tier_poll_thread(self) -> None:
|
|
"""Stop and join the tier-poll thread.
|
|
|
|
Must run before any tier teardown: the thread drives tier transports,
|
|
and closing one underneath it can crash the process rather than raise.
|
|
"""
|
|
self._tier_poll_stop.set()
|
|
thread = self._tier_poll_thread
|
|
if thread is None:
|
|
return
|
|
thread.join(timeout=_TIER_POLL_JOIN_TIMEOUT_S)
|
|
if thread.is_alive():
|
|
logger.error(
|
|
"KV offload tier-poll thread did not exit within %.1fs",
|
|
_TIER_POLL_JOIN_TIMEOUT_S,
|
|
)
|
|
else:
|
|
self._tier_poll_thread = None
|
|
|
|
def _tier_poll_loop(self) -> None:
|
|
stalled_since: float | None = None
|
|
# Start of the current run of failed rounds, and when one last logged.
|
|
failing_since: float | None = None
|
|
last_error_log = 0.0
|
|
while not self._tier_poll_stop.is_set():
|
|
if not self._lock.acquire(timeout=_TIER_POLL_LOCK_TIMEOUT_S):
|
|
now = time.monotonic()
|
|
if stalled_since is None:
|
|
stalled_since = now
|
|
elif now - stalled_since > _TIER_POLL_STALL_WARN_S:
|
|
logger.warning(
|
|
"KV offload tier-poll thread has not acquired the "
|
|
"manager lock for %.0fs; peer requests are only being "
|
|
"served at engine-step boundaries.",
|
|
now - stalled_since,
|
|
)
|
|
stalled_since = None
|
|
continue
|
|
stalled_since = None
|
|
try:
|
|
self.poll_tiers()
|
|
failing_since = None
|
|
except Exception:
|
|
# Never let a round kill the thread silently: on_schedule_end()
|
|
# still serves these tiers, so persistent failure degrades to
|
|
# per-step servicing instead of stopping the tier-poll thread.
|
|
now = time.monotonic()
|
|
if failing_since is None:
|
|
failing_since = now
|
|
if now - last_error_log >= _TIER_POLL_ERROR_LOG_INTERVAL_S:
|
|
last_error_log = now
|
|
logger.exception("KV offload tier-poll round failed")
|
|
finally:
|
|
self._lock.release()
|
|
if (
|
|
failing_since is not None
|
|
and time.monotonic() - failing_since >= _TIER_POLL_GIVE_UP_AFTER_S
|
|
):
|
|
logger.error(
|
|
"KV offload tier-poll thread stopping after %.0fs of "
|
|
"consecutive failures; falling back to per-step servicing.",
|
|
_TIER_POLL_GIVE_UP_AFTER_S,
|
|
)
|
|
return
|
|
# Yield unconditionally, and only after release(). Python locks are
|
|
# not fair, so a release-then-reacquire loop could starve the
|
|
# scheduler thread; this pause is what bounds its wait to one round.
|
|
self._tier_poll_stop.wait(self._tier_poll_interval_s)
|
|
|
|
def poll_tiers(self) -> None:
|
|
"""Advance the control plane of tiers that cannot wait for a step.
|
|
|
|
Polls those tiers for finished jobs and lets them serve whatever their
|
|
counterpart has asked for -- the same two steps on_schedule_end() runs,
|
|
narrowed to the tiers that opted in.
|
|
|
|
Excludes _flush_pending_promotions() and _flush_pending_cascades() by
|
|
design: both are per-step batching points, and running them here would
|
|
fragment batches the scheduler thread accumulated during schedule(). A
|
|
promotion that a lookup() from this path initiates therefore waits for
|
|
the next on_schedule_end() to be submitted.
|
|
|
|
The caller must hold self.lock for the whole call. serve_external_requests()
|
|
relies on that: it establishes lookup() HITs and then pins them, and the
|
|
two must not be separated by an eviction.
|
|
|
|
Serving can look keys up through the parent, and lookup() runs the
|
|
step's once-per-step poll. From here that poll would reach every tier
|
|
off-thread, re-enter the serving tier's get_finished_jobs() from inside
|
|
its own serve, and leave the gate set so the next step skips its poll.
|
|
The round has just polled its tiers itself, so it holds the gate shut
|
|
for its duration and then puts it back untouched.
|
|
"""
|
|
gate = self._processed_jobs_this_step
|
|
self._processed_jobs_this_step = True
|
|
try:
|
|
self._process_finished_jobs(self._polled_tiers)
|
|
for tier in self._polled_tiers:
|
|
tier.serve_external_requests(self._tier_parents[tier])
|
|
finally:
|
|
self._processed_jobs_this_step = gate
|
|
|
|
@property
|
|
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
|
|
return self._jobs
|
|
|
|
def _next_job_id(self) -> JobId:
|
|
"""Generate a unique job ID for async transfer tracking."""
|
|
job_id = self._job_id_counter
|
|
self._job_id_counter += 1
|
|
return job_id
|
|
|
|
def _register_job(self, transfer_job: TransferJob, tier_idx: int) -> None:
|
|
job_metadata = JobMetadata(transfer_job, tier_idx)
|
|
self._jobs[transfer_job.job_id] = job_metadata
|
|
self._metrics.on_job_registered(job_metadata)
|
|
|
|
def _pop_job(self, job_id: JobId) -> JobMetadata | None:
|
|
return self._jobs.pop(job_id, None)
|
|
|
|
def _maybe_process_finished_jobs(self):
|
|
"""Poll secondary tiers for completed jobs (at most once per step).
|
|
|
|
Guarded by _processed_jobs_this_step: the first call in an engine step
|
|
does the actual polling; subsequent calls are no-ops. The flag is reset
|
|
in on_schedule_end() at the end of each step.
|
|
"""
|
|
if self._processed_jobs_this_step:
|
|
return
|
|
self._processed_jobs_this_step = True
|
|
self._process_finished_jobs()
|
|
|
|
def _complete_promotion(
|
|
self, job_metadata: JobMetadata, completed_job: JobResult
|
|
) -> None:
|
|
transfer_job = job_metadata.transfer_job
|
|
successful_keys = completed_job.successful_keys
|
|
failed_keys: Collection[OffloadKey]
|
|
if completed_job.success:
|
|
successful_keys = transfer_job.keys
|
|
failed_keys = ()
|
|
elif successful_keys:
|
|
failed_keys_set = set(transfer_job.keys)
|
|
assert failed_keys_set.issuperset(successful_keys), (
|
|
f"Finished promotion job_id {completed_job.job_id} "
|
|
"reported unknown successful keys"
|
|
)
|
|
failed_keys_set.difference_update(successful_keys)
|
|
failed_keys = failed_keys_set
|
|
else:
|
|
successful_keys = ()
|
|
failed_keys = transfer_job.keys
|
|
|
|
load_sources = self._request_load_sources.get(transfer_job.req_context.req_id)
|
|
if load_sources is not None:
|
|
source = self.secondary_tiers[job_metadata.tier_idx].cache_hit_source
|
|
for key in failed_keys:
|
|
if load_sources.get(key) == source:
|
|
del load_sources[key]
|
|
|
|
if successful_keys:
|
|
self.primary_tier.complete_write(
|
|
successful_keys,
|
|
transfer_job.req_context,
|
|
True,
|
|
)
|
|
if failed_keys:
|
|
self.primary_tier.complete_write(
|
|
failed_keys,
|
|
transfer_job.req_context,
|
|
False,
|
|
)
|
|
|
|
def _process_finished_jobs(
|
|
self,
|
|
tiers: Sequence[SecondaryTierManager] | None = None,
|
|
):
|
|
"""Unconditionally poll secondary tiers for completed jobs.
|
|
|
|
This method:
|
|
1. Calls get_finished_jobs() on each secondary tier
|
|
2. For completed stores (primary→secondary): calls primary.complete_read()
|
|
to decrement ref_cnt
|
|
3. For completed loads (secondary→primary): calls primary.complete_write()
|
|
to make chunks available
|
|
|
|
Args:
|
|
tiers: Tiers to poll. Defaults to every secondary tier. Pass a
|
|
subset to poll only the tiers a caller is responsible for, so
|
|
a tier that expects to be polled on the scheduler thread is
|
|
not dragged elsewhere.
|
|
|
|
"""
|
|
if tiers is None:
|
|
tiers = self.secondary_tiers
|
|
for tier in tiers:
|
|
i = self._tier_index[tier]
|
|
for completed_job in tier.get_finished_jobs():
|
|
job_id = completed_job.job_id
|
|
job_metadata = self._pop_job(job_id)
|
|
assert job_metadata is not None, (
|
|
f"Finished job_id {job_id} from tier #{i}"
|
|
f" ({tier.tier_type}) not in _jobs"
|
|
)
|
|
assert job_metadata.tier_idx == i, (
|
|
f"Finished job_id {job_id} reported by tier #{i}"
|
|
f" but belongs to tier #{job_metadata.tier_idx}"
|
|
)
|
|
transfer_job = job_metadata.transfer_job
|
|
self._metrics.on_job_finished(job_metadata, completed_job)
|
|
|
|
if transfer_job.is_promotion:
|
|
# secondary→primary transfer (promotion) completed.
|
|
# Make chunks available in primary tier.
|
|
self._complete_promotion(job_metadata, completed_job)
|
|
else:
|
|
# primary→secondary transfer completed.
|
|
# Decrement ref_cnt on primary chunks.
|
|
self.primary_tier.complete_read(
|
|
transfer_job.keys, transfer_job.req_context
|
|
)
|
|
if completed_job.success:
|
|
self._update_backpressure(tier, job_metadata, completed_job)
|
|
|
|
def _should_store_to_tier(
|
|
self, tier: SecondaryTierManager, num_blocks: int
|
|
) -> bool:
|
|
detector = tier.bp_detector
|
|
if detector is None:
|
|
return True
|
|
return detector.should_store(num_blocks)
|
|
|
|
def _update_backpressure(
|
|
self,
|
|
tier: SecondaryTierManager,
|
|
job_metadata: JobMetadata,
|
|
completed_job: JobResult,
|
|
) -> None:
|
|
detector = tier.bp_detector
|
|
if detector is None:
|
|
return
|
|
was_under_pressure = detector.is_under_pressure()
|
|
tj = job_metadata.transfer_job
|
|
num_bytes = (
|
|
completed_job.transfer_bytes
|
|
if completed_job.transfer_bytes is not None
|
|
else len(tj.keys) * tier.block_size_bytes
|
|
)
|
|
detector.update(tj.submit_time, num_bytes)
|
|
if detector.is_under_pressure() != was_under_pressure:
|
|
tier_idx = self._tier_index[tier]
|
|
logger.info(
|
|
"Tier #%d (%s) back-pressure %s (stats=%s)",
|
|
tier_idx,
|
|
tier.tier_type,
|
|
"activated" if detector.is_under_pressure() else "cleared",
|
|
detector.stats,
|
|
)
|
|
|
|
@override
|
|
def lookup(
|
|
self,
|
|
key: OffloadKey,
|
|
req_context: ReqContext,
|
|
*,
|
|
exclude_tier_idx: int | None = None,
|
|
) -> LookupResult:
|
|
"""Check whether a single chunk is offloaded and ready.
|
|
|
|
Algorithm:
|
|
1. Process any completed async jobs first.
|
|
2. Query primary tier — short-circuit on hit or in-flight.
|
|
3. On primary miss, query secondary tiers — stop on first
|
|
hit and initiate promotion.
|
|
|
|
Args:
|
|
key: Chunk hash to look up.
|
|
req_context: Per-request context.
|
|
exclude_tier_idx: Skip this tier index during the lookup.
|
|
|
|
Returns:
|
|
HIT — chunk is ready in the primary tier.
|
|
HIT_PENDING — chunk found but not yet readable (write
|
|
in-flight on the primary tier).
|
|
RETRY — promotion started or a secondary tier is busy.
|
|
MISS — chunk not found in any tier, or primary is full
|
|
and cannot accept a promotion.
|
|
|
|
"""
|
|
# Poll first so a promotion that finished since the last call is
|
|
# already reflected as HIT (not stale HIT_PENDING/MISS) below, and
|
|
# so chunks freed by cascade or promotion completions are evictable
|
|
# in time for a promotion this lookup may initiate.
|
|
self._maybe_process_finished_jobs()
|
|
|
|
start_time = time.monotonic()
|
|
primary_hit = self.primary_tier.lookup(key, req_context)
|
|
lookup_duration = time.monotonic() - start_time
|
|
self._metrics.on_lookup(
|
|
req_context,
|
|
key,
|
|
self._metrics.primary_tier_label,
|
|
primary_hit,
|
|
lookup_duration,
|
|
)
|
|
if primary_hit is LookupResult.HIT:
|
|
return LookupResult.HIT
|
|
if primary_hit is LookupResult.HIT_PENDING:
|
|
return LookupResult.HIT_PENDING
|
|
|
|
any_retry = False
|
|
for i, tier in enumerate(self.secondary_tiers):
|
|
if i == exclude_tier_idx:
|
|
continue
|
|
if not req_context.load_tier_filter.allows(tier.medium, tier.locality):
|
|
continue
|
|
labelvalues = self._metrics.tier_label(i)
|
|
start_time = time.monotonic()
|
|
result = tier.lookup(key, req_context)
|
|
lookup_duration = time.monotonic() - start_time
|
|
if result is LookupResult.HIT:
|
|
self._metrics.on_lookup(
|
|
req_context,
|
|
key,
|
|
labelvalues,
|
|
result,
|
|
lookup_duration,
|
|
)
|
|
promoted = self._initiate_promotion(i, key, req_context)
|
|
return LookupResult.MISS if not promoted else LookupResult.HIT_PENDING
|
|
if result is LookupResult.RETRY:
|
|
any_retry = True
|
|
self._metrics.on_lookup(
|
|
req_context,
|
|
key,
|
|
labelvalues,
|
|
result,
|
|
lookup_duration,
|
|
)
|
|
|
|
if any_retry:
|
|
return LookupResult.RETRY
|
|
return LookupResult.MISS
|
|
|
|
@override
|
|
def get_load_source(
|
|
self, key: OffloadKey, req_context: ReqContext
|
|
) -> CacheHitSource:
|
|
load_sources = self._request_load_sources.get(req_context.req_id)
|
|
if load_sources is not None and key in load_sources:
|
|
return load_sources[key]
|
|
return self.primary_tier.get_load_source(key, req_context)
|
|
|
|
def _initiate_promotion(
|
|
self,
|
|
tier_idx: int,
|
|
key: OffloadKey,
|
|
req_context: ReqContext,
|
|
) -> bool:
|
|
"""Queue a chunk for promotion from a secondary tier to the primary tier.
|
|
|
|
Allocates space in the primary tier immediately (sets ref_cnt=-1 so
|
|
subsequent lookups within the same step see the slot as in-flight),
|
|
then defers the actual submit_load() call to _flush_pending_promotions()
|
|
so all chunks queued during one engine step are submitted as a single
|
|
batched job.
|
|
|
|
Args:
|
|
tier_idx: The secondary tier index to promote from
|
|
key: Chunk to promote
|
|
req_context: Per-request context forwarded to primary.prepare_write().
|
|
|
|
Returns:
|
|
True if promotion was initiated, False if primary tier is full.
|
|
|
|
"""
|
|
# Allocate space in primary tier for promoted chunk.
|
|
# Must happen immediately so primary.lookup() returns None (in-flight)
|
|
# for this key on any subsequent lookup() call within the same step,
|
|
# preventing duplicate promotion attempts.
|
|
primary_write_result = self.primary_tier.prepare_write([key], req_context)
|
|
|
|
if primary_write_result is None:
|
|
# Primary tier is full; caller should treat the chunk as unavailable
|
|
# rather than retrying indefinitely.
|
|
self._metrics.on_promotion_allocation_failure()
|
|
return False
|
|
|
|
store_spec = primary_write_result.store_spec
|
|
assert isinstance(store_spec, CPULoadStoreSpec)
|
|
load_sources = self._request_load_sources.setdefault(req_context.req_id, {})
|
|
source = self.secondary_tiers[tier_idx].cache_hit_source
|
|
for promoted_key in primary_write_result.keys_to_store:
|
|
load_sources[promoted_key] = source
|
|
# Defer submit_load to on_schedule_end(). Group by (tier, request) so
|
|
# each request's chunks are submitted as one batched job per tier.
|
|
tier_pending = self._pending_load_submissions.setdefault(tier_idx, {})
|
|
ctx_id = req_context.req_id
|
|
if ctx_id not in tier_pending:
|
|
tier_pending[ctx_id] = PendingPromotion(
|
|
keys=[], chunk_ids=[], req_context=req_context
|
|
)
|
|
entry = tier_pending[ctx_id]
|
|
entry.keys.extend(primary_write_result.keys_to_store)
|
|
entry.chunk_ids.extend(store_spec.chunk_ids)
|
|
return True
|
|
|
|
def _flush_pending_promotions(self) -> None:
|
|
"""Submit one batched submit_load() per (tier, request).
|
|
|
|
Called from on_schedule_end() at the end of each scheduler step,
|
|
flushing all promotion requests deferred during lookup().
|
|
"""
|
|
if not self._pending_load_submissions:
|
|
return
|
|
|
|
for tier_idx, pending_by_ctx in self._pending_load_submissions.items():
|
|
tier = self.secondary_tiers[tier_idx]
|
|
for entry in pending_by_ctx.values():
|
|
job_id = self._next_job_id()
|
|
job_metadata = TransferJob(
|
|
job_id=job_id,
|
|
keys=entry.keys,
|
|
chunk_ids=np.array(entry.chunk_ids, dtype=np.int32),
|
|
is_promotion=True,
|
|
req_context=entry.req_context,
|
|
)
|
|
self._register_job(job_metadata, tier_idx)
|
|
tier.submit_load(job_metadata)
|
|
|
|
self._pending_load_submissions.clear()
|
|
|
|
@override
|
|
def prepare_load(
|
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
|
) -> LoadStoreSpec:
|
|
"""Prepare chunks to be loaded from primary tier to GPU.
|
|
|
|
Callers only pass keys already confirmed HIT by lookup() earlier this
|
|
step.
|
|
|
|
This increments ref_cnt on the chunks in the primary tier, protecting
|
|
them from eviction during the transfer.
|
|
|
|
Args:
|
|
keys: Chunks to prepare for loading.
|
|
req_context: Per-request context.
|
|
|
|
Returns:
|
|
LoadStoreSpec for reading from primary tier.
|
|
|
|
"""
|
|
return self.primary_tier.prepare_load(keys, req_context)
|
|
|
|
@override
|
|
def touch(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
|
"""Mark chunks as recently used in all tiers.
|
|
|
|
Args:
|
|
keys: Chunks to mark as recently used.
|
|
req_context: Per-request context.
|
|
|
|
"""
|
|
self.primary_tier.touch(keys, req_context)
|
|
for tier in self.secondary_tiers:
|
|
tier.touch(keys, req_context)
|
|
|
|
@override
|
|
def complete_load(self, keys: Collection[OffloadKey], req_context: ReqContext):
|
|
"""Mark chunks as done loading from primary tier to GPU.
|
|
|
|
This decrements ref_cnt on the chunks in the primary tier, allowing
|
|
them to be evicted again.
|
|
|
|
Args:
|
|
keys: Chunks that finished loading.
|
|
req_context: Per-request context.
|
|
|
|
"""
|
|
self.primary_tier.complete_load(keys, req_context)
|
|
|
|
@override
|
|
def prepare_store(
|
|
self, keys: Collection[OffloadKey], req_context: ReqContext
|
|
) -> PrepareStoreOutput | None:
|
|
"""Prepare chunks to be stored from GPU to primary tier.
|
|
|
|
CRITICAL: This method polls for finished jobs FIRST to ensure that any
|
|
completed async transfers have their ref_cnt decremented before the
|
|
primary tier makes eviction decisions.
|
|
|
|
For request-level tiers, chunks already present in the primary tier
|
|
are immediately cascaded via submit_store().
|
|
|
|
Args:
|
|
keys: Chunks to prepare for storing.
|
|
req_context: Per-request context.
|
|
|
|
Returns:
|
|
PrepareStoreOutput describing where to store chunks and what was
|
|
evicted, or None if store cannot proceed.
|
|
|
|
"""
|
|
# Step 1: Poll for completed async jobs FIRST
|
|
# _process_finished_jobs() handles two kinds of completions here:
|
|
# - Cascade completions (store to a secondary tier, either a local
|
|
# cascade or a store job created for a remote requester via
|
|
# create_store_job()): decrements ref_cnt on the primary chunks
|
|
# that were read, making them evictable again once ref_cnt hits 0.
|
|
# - Promotion completions (secondary->primary loads): sets a
|
|
# not-yet-ready chunk's ref_cnt from -1 to 0 via complete_write(),
|
|
# making it evictable for the first time.
|
|
# Both must be accounted for before the eviction decision below.
|
|
# Unconditional, not _maybe_process_finished_jobs(): eviction must see
|
|
# the freshest completions, and on_schedule_end now runs at the very end
|
|
# of the step, so the once-per-step gate would otherwise still be set by
|
|
# this step's first lookup() and skip the poll.
|
|
self._processed_jobs_this_step = True
|
|
self._process_finished_jobs()
|
|
|
|
# Step 2: Store to primary tier (new chunks only).
|
|
# Cascading of these newly-stored chunks to ALL secondary tiers
|
|
# happens later in complete_store(), after the GPU→Primary transfer
|
|
# completes.
|
|
primary_result = self.primary_tier.prepare_store(keys, req_context)
|
|
|
|
if primary_result is None:
|
|
return None
|
|
|
|
if primary_result.keys_to_store:
|
|
state = self._req_state[req_context.req_id]
|
|
state.pending_primary_stores += 1
|
|
|
|
# Step 3: For request-level tiers, cascade chunks already in primary
|
|
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
|
|
if request_level_tiers:
|
|
keys_to_store_set = set(primary_result.keys_to_store)
|
|
keys_already_in_primary = tuple(
|
|
k for k in keys if k not in keys_to_store_set
|
|
)
|
|
if keys_already_in_primary:
|
|
self._cascade_existing_chunks_to_request_level_tiers(
|
|
keys_already_in_primary, req_context, request_level_tiers
|
|
)
|
|
|
|
return primary_result
|
|
|
|
def _cascade_existing_chunks_to_request_level_tiers(
|
|
self,
|
|
keys: Sequence[OffloadKey],
|
|
req_context: ReqContext,
|
|
request_level_tiers: set[int],
|
|
) -> None:
|
|
"""For tiers that requested request-level policy, submit_store() for
|
|
chunks that are already present in the primary tier.
|
|
|
|
A key whose primary write is still in flight (HIT_PENDING) cannot be
|
|
dropped: prepare_store already excluded it as present, and the
|
|
scheduler advances past its chunk, so no path offers it again. Park it
|
|
instead. MISS keys are dropped, since nothing is there to read.
|
|
|
|
The primary tier resolves every key it holds, so RETRY cannot reach
|
|
here. Parking on it would have no guarantee of ever draining, which is
|
|
what makes parking HIT_PENDING safe, so it is rejected rather than
|
|
guessed at.
|
|
"""
|
|
state = self._req_state[req_context.req_id]
|
|
ready_keys = []
|
|
for key in keys:
|
|
result = self.primary_tier.lookup(key, req_context)
|
|
if result is LookupResult.HIT:
|
|
ready_keys.append(key)
|
|
elif result is LookupResult.HIT_PENDING:
|
|
state.pending_cascade_keys.append(key)
|
|
else:
|
|
assert result is LookupResult.MISS, (
|
|
f"primary tier returned {result} for a cascade key"
|
|
)
|
|
if not ready_keys:
|
|
return
|
|
|
|
for tier_idx in request_level_tiers:
|
|
tier = self.secondary_tiers[tier_idx]
|
|
if not self._should_store_to_tier(tier, len(ready_keys)):
|
|
continue
|
|
job_metadata = self.create_store_job(ready_keys, req_context, tier_idx)
|
|
tier.submit_store(job_metadata)
|
|
|
|
def _flush_pending_cascades(self) -> None:
|
|
"""Retry request-level cascades parked on an in-flight primary write.
|
|
|
|
A parked key always resolves, to HIT or to MISS, so the set drains and
|
|
a request cannot be held from finalization forever.
|
|
"""
|
|
for req_id, state in list(self._req_state.items()):
|
|
if not state.pending_cascade_keys:
|
|
continue
|
|
assert state.request_level_tiers
|
|
keys, state.pending_cascade_keys = state.pending_cascade_keys, []
|
|
self._cascade_existing_chunks_to_request_level_tiers(
|
|
keys, state.req_context, state.request_level_tiers
|
|
)
|
|
self._maybe_finalize_request(req_id)
|
|
|
|
@override
|
|
def complete_store(
|
|
self,
|
|
keys: Collection[OffloadKey],
|
|
req_context: ReqContext,
|
|
success: bool = True,
|
|
) -> None:
|
|
"""Mark chunks as done storing from GPU to primary tier.
|
|
|
|
This is where secondary tier cascading happens — after chunks are
|
|
confirmed to be in the primary tier, they are cascaded to ALL
|
|
secondary tiers.
|
|
|
|
For each secondary tier:
|
|
1. Call primary.prepare_read() to get LoadStoreSpec AND increment
|
|
ref_cnt (protecting chunks during async transfer)
|
|
2. Call tier.submit_store() to start async transfer: primary→secondary
|
|
3. Track the job in _store_jobs dictionary
|
|
|
|
Args:
|
|
keys: Chunks that finished storing.
|
|
success: Whether the GPU→primary transfer succeeded.
|
|
req_context: Per-request context forwarded to primary.prepare_read().
|
|
|
|
"""
|
|
# Step 1: Complete store in primary tier (makes chunks loadable)
|
|
self.primary_tier.complete_store(keys, req_context, success)
|
|
|
|
if success:
|
|
# Step 2: Cascade to ALL secondary tiers
|
|
# For each secondary tier, call primary.prepare_read() to get the
|
|
# LoadStoreSpec AND to increment ref_cnt (protecting chunks from
|
|
# eviction during the async transfer). One prepare_read() call per
|
|
# secondary tier.
|
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
|
if not self._should_store_to_tier(tier, len(keys)):
|
|
continue
|
|
job_metadata = self.create_store_job(keys, req_context, tier_idx)
|
|
tier.submit_store(job_metadata)
|
|
|
|
# Note: The async transfers are now in flight. Their completion is
|
|
# tracked via get_finished_jobs() / _maybe_process_finished_jobs().
|
|
req_id = req_context.req_id
|
|
state = self._req_state[req_id]
|
|
assert state.pending_primary_stores > 0
|
|
state.pending_primary_stores -= 1
|
|
self._maybe_finalize_request(req_id)
|
|
|
|
def create_store_job(
|
|
self,
|
|
keys: Collection[OffloadKey],
|
|
req_context: ReqContext,
|
|
tier_idx: int = 0,
|
|
) -> TransferJob:
|
|
"""Pin chunks in the primary tier and create a tracked store job.
|
|
|
|
Calls prepare_read() to increment ref_cnt (protecting chunks
|
|
from eviction during the async transfer), allocates a job ID,
|
|
and registers the job in _jobs.
|
|
|
|
The caller is responsible for the actual data transfer and
|
|
reporting completion via get_finished_jobs().
|
|
"""
|
|
primary_chunks_spec = self.primary_tier.prepare_read(keys, req_context)
|
|
assert isinstance(primary_chunks_spec, CPULoadStoreSpec)
|
|
job_id = self._next_job_id()
|
|
job_metadata = TransferJob(
|
|
job_id=job_id,
|
|
keys=keys,
|
|
chunk_ids=primary_chunks_spec.chunk_ids,
|
|
is_promotion=False,
|
|
req_context=req_context,
|
|
)
|
|
self._register_job(job_metadata, tier_idx)
|
|
return job_metadata
|
|
|
|
@override
|
|
def on_new_request(
|
|
self,
|
|
req_context: ReqContext,
|
|
*,
|
|
exclude_tier_idx: int | None = None,
|
|
) -> RequestOffloadingContext:
|
|
"""Query each secondary tier for its offload policy preference.
|
|
|
|
Returns REQUEST_LEVEL if ANY secondary tier wants request-level.
|
|
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
|
|
"""
|
|
state = RequestState(req_context=req_context)
|
|
self._metrics.on_new_request(req_context)
|
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
|
if tier_idx == exclude_tier_idx:
|
|
continue
|
|
tier_ctx = tier.on_new_request(req_context)
|
|
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
|
|
if state.request_level_tiers is None:
|
|
state.request_level_tiers = set()
|
|
state.request_level_tiers.add(tier_idx)
|
|
self._req_state[req_context.req_id] = state
|
|
|
|
policy = (
|
|
OffloadPolicy.REQUEST_LEVEL
|
|
if state.request_level_tiers
|
|
else OffloadPolicy.CHUNK_LEVEL
|
|
)
|
|
return RequestOffloadingContext(policy=policy)
|
|
|
|
@override
|
|
def on_request_finished(
|
|
self,
|
|
req_context: ReqContext,
|
|
*,
|
|
exclude_tier_idx: int | None = None,
|
|
) -> None:
|
|
self.primary_tier.on_request_finished(req_context)
|
|
state = self._req_state[req_context.req_id]
|
|
state.is_finished = True
|
|
self._maybe_finalize_request(req_context.req_id, exclude_tier_idx)
|
|
|
|
def _maybe_finalize_request(
|
|
self,
|
|
req_id: str,
|
|
exclude_tier_idx: int | None = None,
|
|
) -> None:
|
|
"""Finalize secondary tiers once no more cascades can be submitted.
|
|
|
|
Their finalization is delayed until pending GPU->primary stores
|
|
finish, since those callbacks may still submit secondary stores.
|
|
"""
|
|
state = self._req_state[req_id]
|
|
if not state.is_finished:
|
|
return
|
|
if state.pending_primary_stores != 0:
|
|
return
|
|
if state.pending_cascade_keys:
|
|
return
|
|
|
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
|
if tier_idx == exclude_tier_idx:
|
|
continue
|
|
tier.on_request_finished(state.req_context)
|
|
self._metrics.on_request_finished(state.req_context)
|
|
self._request_load_sources.pop(req_id, None)
|
|
del self._req_state[req_id]
|
|
|
|
@override
|
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
|
"""End-of-schedule hook: process finished jobs, flush deferred
|
|
promotions, and reset the per-step gate.
|
|
|
|
Called once per scheduler step, as the last manager call of the step,
|
|
from OffloadingConnectorScheduler.build_connector_meta().
|
|
"""
|
|
# Catch-all poll: guarantees jobs are processed even on steps where
|
|
# lookup()/prepare_store() were never called (e.g. no requests
|
|
# scheduled but a tier still has_pending_work()).
|
|
self._maybe_process_finished_jobs()
|
|
|
|
for tier in self.secondary_tiers:
|
|
tier.serve_external_requests(self._tier_parents[tier])
|
|
|
|
# Reset the per-step gate AFTER serve_external_requests so that
|
|
# lookup() calls within it skip redundant _process_finished_jobs().
|
|
self._processed_jobs_this_step = False
|
|
|
|
self._flush_pending_promotions()
|
|
# Keys parked by this step's prepare_store cannot have completed yet, so
|
|
# they are simply re-parked here and retried on the next step. That is
|
|
# safe: _maybe_finalize_request() holds off while pending_cascade_keys is
|
|
# non-empty, so nothing finalizes early and the list still drains.
|
|
self._flush_pending_cascades()
|
|
for tier in self.secondary_tiers:
|
|
tier.on_schedule_end(context)
|
|
|
|
for req_id in context.new_req_ids:
|
|
state = self._req_state.get(req_id)
|
|
if state is None:
|
|
continue
|
|
self._metrics.on_request_allocated(state.req_context)
|
|
|
|
@override
|
|
def has_pending_work(self) -> bool:
|
|
# In-flight primary<->secondary transfers (pending promotions are
|
|
# translated to transfer jobs in on_schedule_end), plus any work the
|
|
# secondary tiers themselves still have outstanding.
|
|
return (
|
|
bool(self._jobs)
|
|
or any(state.pending_cascade_keys for state in self._req_state.values())
|
|
or any(tier.has_pending_work() for tier in self.secondary_tiers)
|
|
)
|
|
|
|
@override
|
|
def take_events(self) -> Iterable[OffloadingEvent]:
|
|
"""Yield events owned by the primary and secondary tiers.
|
|
|
|
Yields:
|
|
New OffloadingEvents collected by each tier since the last call.
|
|
|
|
"""
|
|
yield from self.primary_tier.take_events()
|
|
for tier in self.secondary_tiers:
|
|
yield from tier.take_events()
|
|
|
|
@override
|
|
def reset_cache(self) -> None:
|
|
"""Reset transfer bookkeeping and primary-tier cache.
|
|
|
|
Called during sleep, weight update, or resume. Each secondary tier
|
|
drains its in-flight transfers via drain_jobs() so no tier I/O is
|
|
touching primary memory before the primary tier is reset. A stuck
|
|
tier will block here visibly — preferable to silent corruption
|
|
from reusing primary slots while a transfer is mid-copy.
|
|
|
|
Secondary tiers are intentionally not reset: persistent stores
|
|
(FS, network) keep their data across resets. Active request state is
|
|
retained so those requests can continue after the reset; finished
|
|
requests are finalized and removed.
|
|
"""
|
|
for tier in self.secondary_tiers:
|
|
tier.drain_jobs()
|
|
# All tier I/O has stopped; consume their completion notifications
|
|
# so manager bookkeeping is consistent before the primary reset.
|
|
self._process_finished_jobs()
|
|
assert not self._jobs
|
|
|
|
# Deferred promotion submissions reserve primary slots that the
|
|
# reset below invalidates; their submit_load() has not yet been
|
|
# called so no tier I/O is touching that memory.
|
|
self._pending_load_submissions.clear()
|
|
self._request_load_sources.clear()
|
|
self._metrics.assert_idle()
|
|
|
|
finished_req_ids = []
|
|
for req_id, state in self._req_state.items():
|
|
state.pending_primary_stores = 0
|
|
state.pending_cascade_keys.clear()
|
|
if not state.is_finished:
|
|
continue
|
|
for tier in self.secondary_tiers:
|
|
tier.on_request_finished(state.req_context)
|
|
self._metrics.on_request_finished(state.req_context)
|
|
finished_req_ids.append(req_id)
|
|
|
|
self.primary_tier.reset_cache()
|
|
|
|
for req_id in finished_req_ids:
|
|
del self._req_state[req_id]
|
|
self._processed_jobs_this_step = False
|
|
|
|
for tier in self.secondary_tiers:
|
|
if tier.bp_detector is not None:
|
|
tier.bp_detector.reset()
|
|
|
|
@override
|
|
def get_stats(self) -> OffloadingConnectorStats | None:
|
|
stats = self.primary_tier.get_stats()
|
|
|
|
if stats is not None and stats.is_empty():
|
|
stats = None
|
|
|
|
self._metrics.record_backpressure(self.secondary_tiers)
|
|
metrics_stats = self._metrics.take_stats()
|
|
if metrics_stats is not None:
|
|
if stats is None:
|
|
stats = metrics_stats
|
|
else:
|
|
stats.aggregate(metrics_stats)
|
|
|
|
for tier in self.secondary_tiers:
|
|
tier_stats = tier.get_stats()
|
|
if tier_stats is None or tier_stats.is_empty():
|
|
continue
|
|
if stats is None:
|
|
stats = tier_stats
|
|
else:
|
|
stats.aggregate(tier_stats)
|
|
|
|
return stats
|
|
|
|
@override
|
|
def config_info(self) -> Sequence[ConfigInfoMapping]:
|
|
"""Return one mapping for each tier, primary tier first, with TIER_LABEL.
|
|
|
|
Raises:
|
|
ValueError: If the primary tier returns more than one mapping.
|
|
|
|
"""
|
|
(primary_info,) = self.primary_tier.config_info()
|
|
primary_tier_info = dict(primary_info)
|
|
primary_tier_info[TIER_LABEL] = self._metrics.primary_tier_label[0]
|
|
tier_infos = [primary_tier_info]
|
|
|
|
for tier in self.secondary_tiers:
|
|
tier_info = dict(tier.config_info())
|
|
tier_info[TIER_LABEL] = self._metrics.tier_label(self._tier_index[tier])[0]
|
|
tier_infos.append(tier_info)
|
|
|
|
return tier_infos
|
|
|
|
@override
|
|
def shutdown(self) -> None:
|
|
"""Shut down secondary tiers before releasing primary resources.
|
|
|
|
Stops the tier-poll thread first: it drives tier transports, and
|
|
tearing one down underneath it (closing a socket, destroying a ZMQ
|
|
context) can take the process down rather than raise. If the thread
|
|
will not exit, hold the manager lock across tier teardown so it cannot
|
|
be mid-round, and raise if even that is unavailable -- tearing a tier
|
|
down under a live round risks a crash, and skipping it silently would
|
|
leak the primary's shared memory region.
|
|
|
|
Unlike every other manager method, call this WITHOUT holding
|
|
self.lock: it takes the lock itself, which is not reentrant, and the
|
|
tier-poll thread needs the lock free to notice the stop request
|
|
and exit. A caller that does hold it only gets the RuntimeError above,
|
|
after waiting out the timeout.
|
|
|
|
Every secondary tier is given a shutdown attempt. If any shutdown
|
|
fails, preserve the primary mmap because a failed tier may still use it.
|
|
"""
|
|
self._stop_tier_poll_thread()
|
|
if self._tier_poll_thread is not None:
|
|
if not self._lock.acquire(timeout=_TIER_POLL_JOIN_TIMEOUT_S):
|
|
raise RuntimeError(
|
|
"KV offload tier-poll thread is still running and the "
|
|
"manager lock is unavailable; secondary tiers were not "
|
|
"shut down. shutdown() must be called without holding "
|
|
"manager.lock."
|
|
)
|
|
try:
|
|
self._shutdown_tiers()
|
|
finally:
|
|
self._lock.release()
|
|
return
|
|
self._shutdown_tiers()
|
|
|
|
def _shutdown_tiers(self) -> None:
|
|
shutdown_error: Exception | None = None
|
|
for tier_idx, tier in enumerate(self.secondary_tiers):
|
|
try:
|
|
tier.shutdown()
|
|
except Exception as exc:
|
|
shutdown_error = exc
|
|
logger.exception(
|
|
"Failed to shut down secondary tier #%d "
|
|
"(tier_type=%s, impl_class=%s)",
|
|
tier_idx,
|
|
tier.tier_type,
|
|
type(tier).__name__,
|
|
)
|
|
|
|
if shutdown_error is not None:
|
|
raise shutdown_error
|
|
|
|
self.primary_tier.shutdown()
|