[KV Offload] Expose OffloadingManager.lock()/unlock() and service the tiering control plane off the scheduler thread (#58168)
Signed-off-by: Liran Schour <lirans@il.ibm.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Kevin H. Luu
parent
c54dca96bd
commit
c5f8adc0b1
@@ -2,7 +2,8 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import time
|
||||
from collections import Counter
|
||||
from collections.abc import Iterable, Sequence
|
||||
from collections.abc import Iterable, Iterator, Sequence
|
||||
from contextlib import ExitStack, contextmanager
|
||||
from dataclasses import dataclass, field, replace
|
||||
from itertools import chain, islice
|
||||
from typing import Any, NamedTuple
|
||||
@@ -640,6 +641,71 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
|
||||
|
||||
# Holds the manager lock for the current schedule(), or None between steps.
|
||||
# Read and written only by the scheduler thread, so it needs no
|
||||
# synchronization of its own. Declared on the class so an instance built
|
||||
# without __init__ still reads None rather than raising.
|
||||
_step_region: ExitStack | None = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Manager exclusion
|
||||
#
|
||||
# The manager is entered from the scheduler thread and, for managers that
|
||||
# run one, from a tier-poll thread. Manager methods never take their
|
||||
# own lock, so the exclusion boundary lives here. manager.lock may be a
|
||||
# no-op for a manager that needs no exclusion; it is entered regardless.
|
||||
#
|
||||
# schedule() holds one region for the whole step: opened by the first
|
||||
# manager-touching hook and closed at the end of build_connector_meta.
|
||||
# That span is required, not tidiness -- get_num_new_matched_tokens
|
||||
# establishes lookup() HITs that update_state_after_alloc later pins with
|
||||
# prepare_load(), and prepare_store() evicts after both. Releasing between
|
||||
# them would let another thread evict a block the step already treated as
|
||||
# present.
|
||||
#
|
||||
# Hooks outside schedule() use a short region instead, so the manager is
|
||||
# free while the engine runs the model.
|
||||
#
|
||||
# The step region outlives any single call, so it cannot be a with-block;
|
||||
# an ExitStack carries the entered lock from open to close instead.
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _acquire_step(self) -> None:
|
||||
"""Open the step-scoped region. Idempotent within a step."""
|
||||
if self._step_region is not None:
|
||||
return
|
||||
region = ExitStack()
|
||||
region.enter_context(self.manager.lock)
|
||||
self._step_region = region
|
||||
|
||||
def _release_step(self) -> None:
|
||||
"""Close the step-scoped region, letting other threads in."""
|
||||
region = self._step_region
|
||||
if region is None:
|
||||
return
|
||||
self._step_region = None
|
||||
region.close()
|
||||
|
||||
@contextmanager
|
||||
def _manager_locked(self) -> Iterator[None]:
|
||||
"""Hold the manager for the duration of one hook outside schedule().
|
||||
|
||||
None of these hooks can legitimately run inside the step-scoped region:
|
||||
on_new_request comes from add_request, has_pending_push_work from the
|
||||
engine's liveness check before schedule(), the rest from output
|
||||
processing or a client RPC. So finding the region open here means a
|
||||
previous schedule() raised between _acquire_step() and the release in
|
||||
build_connector_meta. Recover instead of deadlocking.
|
||||
"""
|
||||
if self._step_region is not None:
|
||||
logger.error(
|
||||
"Offloading manager lock was left held by a previous "
|
||||
"schedule(); releasing it before continuing."
|
||||
)
|
||||
self._release_step()
|
||||
with self.manager.lock:
|
||||
yield
|
||||
|
||||
def _maybe_observe_lookup_async_delay(
|
||||
self, req_status: RequestOffloadState
|
||||
) -> None:
|
||||
@@ -1031,7 +1097,8 @@ class OffloadingConnectorScheduler:
|
||||
def on_new_request(self, request: Request) -> None:
|
||||
"""Called when a new request is added to the scheduler."""
|
||||
req_context = _create_req_context(request)
|
||||
offloading_context = self.manager.on_new_request(req_context)
|
||||
with self._manager_locked():
|
||||
offloading_context = self.manager.on_new_request(req_context)
|
||||
req_status = RequestOffloadState(
|
||||
config=self.config,
|
||||
req=request,
|
||||
@@ -1067,6 +1134,10 @@ class OffloadingConnectorScheduler:
|
||||
(between scheduler steps).
|
||||
|
||||
"""
|
||||
# First manager-touching hook of a step: opens the step-scoped region,
|
||||
# closed at the end of build_connector_meta.
|
||||
self._acquire_step()
|
||||
|
||||
req_status = self._req_status[request.request_id]
|
||||
for group_state in req_status.group_states:
|
||||
group_state.block_ids.clear()
|
||||
@@ -1161,6 +1232,10 @@ class OffloadingConnectorScheduler:
|
||||
if num_external_tokens == 0:
|
||||
return
|
||||
|
||||
# prepare_load() below pins keys that get_num_new_matched_tokens()
|
||||
# already saw as HIT, so both must sit inside the same region.
|
||||
self._acquire_step()
|
||||
|
||||
req_status = self._req_status[request.request_id]
|
||||
|
||||
num_locally_computed_tokens = req_status.num_locally_computed_tokens
|
||||
@@ -1853,13 +1928,24 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
# A step that scheduled nothing reaches build_connector_meta without
|
||||
# any earlier manager touch, so open the region here too.
|
||||
self._acquire_step()
|
||||
try:
|
||||
return self._build_connector_meta(scheduler_output)
|
||||
finally:
|
||||
# The step is over: hand the manager back.
|
||||
self._release_step()
|
||||
|
||||
def _build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
self._update_req_states(scheduler_output)
|
||||
schedule_end_context = ScheduleEndContext(
|
||||
new_req_ids=[req.req_id for req in scheduler_output.scheduled_new_reqs],
|
||||
preempted_req_ids=scheduler_output.preempted_req_ids or (),
|
||||
)
|
||||
self.manager.on_schedule_end(schedule_end_context)
|
||||
|
||||
# Flush jobs for preempted requests.
|
||||
for req_id in scheduler_output.preempted_req_ids or ():
|
||||
@@ -1905,6 +1991,12 @@ class OffloadingConnectorScheduler:
|
||||
self._current_batch_load_jobs = {}
|
||||
self._current_batch_jobs_to_flush = set()
|
||||
self._current_batch_allocated_block_ids = set()
|
||||
|
||||
# Last manager call of the step, after every prepare_store and
|
||||
# on_request_finished above. on_schedule_end is documented as running at
|
||||
# the end of the step, and callers rely on that to bound the step's
|
||||
# exclusive access to the manager.
|
||||
self.manager.on_schedule_end(schedule_end_context)
|
||||
return meta
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
@@ -1912,8 +2004,17 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
While True, build_connector_meta() and update_connector_output()
|
||||
continue to be called even when no requests are scheduled.
|
||||
|
||||
Must use a short region, never the step-scoped one. The engine asks this
|
||||
before schedule() and returns early when the answer is False, so a
|
||||
step-scoped acquire here would never reach build_connector_meta's
|
||||
release and would hold the manager for good -- which is precisely the
|
||||
state an idle engine settles into.
|
||||
"""
|
||||
return bool(self._jobs) or self.manager.has_pending_work()
|
||||
if self._jobs:
|
||||
return True
|
||||
with self._manager_locked():
|
||||
return self.manager.has_pending_work()
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
"""Update KVConnector state from worker-side connectors output.
|
||||
@@ -1923,6 +2024,10 @@ class OffloadingConnectorScheduler:
|
||||
connectors output.
|
||||
|
||||
"""
|
||||
with self._manager_locked():
|
||||
self._update_connector_output(connector_output)
|
||||
|
||||
def _update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, OffloadingWorkerMetadata):
|
||||
assert meta is None
|
||||
@@ -2001,7 +2106,8 @@ class OffloadingConnectorScheduler:
|
||||
stats = self._connector_stats
|
||||
self._connector_stats = OffloadingConnectorStats()
|
||||
|
||||
manager_stats = self.manager.get_stats()
|
||||
with self._manager_locked():
|
||||
manager_stats = self.manager.get_stats()
|
||||
if manager_stats is not None:
|
||||
if stats is None:
|
||||
stats = manager_stats
|
||||
@@ -2038,8 +2144,9 @@ class OffloadingConnectorScheduler:
|
||||
# Untracked request (offloading never started): no in-flight jobs,
|
||||
# nothing was deferred, so finalize immediately.
|
||||
req_context = _create_req_context(request)
|
||||
self.manager.on_new_request(req_context)
|
||||
self.manager.on_request_finished(req_context)
|
||||
with self._manager_locked():
|
||||
self.manager.on_new_request(req_context)
|
||||
self.manager.on_request_finished(req_context)
|
||||
return False, None
|
||||
|
||||
self._maybe_observe_lookup_async_delay(req_status)
|
||||
@@ -2071,7 +2178,16 @@ class OffloadingConnectorScheduler:
|
||||
the underlying :class:`OffloadingEvent` stream.
|
||||
|
||||
"""
|
||||
yield from self._events_tracker.take_events(self.manager.take_events())
|
||||
# Drain the manager eagerly, inside the region. manager.take_events() is
|
||||
# itself a generator whose backing list is cleared once exhausted, so
|
||||
# iterating it lazily would both hold the manager for as long as the
|
||||
# consumer takes and let a concurrent writer append events that the
|
||||
# clear then discards unpublished.
|
||||
with self._manager_locked():
|
||||
manager_events = list(self.manager.take_events())
|
||||
# Still a generator, so callers testing the result for truthiness keep
|
||||
# seeing a non-empty object the way they do today.
|
||||
yield from self._events_tracker.take_events(manager_events)
|
||||
|
||||
def reset_cache(self) -> None:
|
||||
"""Reset the offloading manager cache, evicting all stored chunks."""
|
||||
@@ -2080,6 +2196,13 @@ class OffloadingConnectorScheduler:
|
||||
assert not self._current_batch_jobs_to_flush
|
||||
assert not self._current_batch_allocated_block_ids
|
||||
|
||||
# Held for the whole reset. manager.reset_cache() drains tiers by
|
||||
# polling them itself, so a tier-poll thread locked out here loses
|
||||
# nothing; it simply skips rounds until the reset completes.
|
||||
with self._manager_locked():
|
||||
self._reset_cache()
|
||||
|
||||
def _reset_cache(self) -> None:
|
||||
# Flush all in-flight jobs
|
||||
self._current_batch_jobs_to_flush.update(self._jobs.keys())
|
||||
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable, Mapping, Sequence
|
||||
from contextlib import AbstractContextManager, nullcontext
|
||||
from dataclasses import dataclass, field, fields
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, NamedTuple, NewType, TypeVar
|
||||
@@ -198,6 +199,16 @@ The class provides the following primitives:
|
||||
as well as a list of blocks that were evicted as a result.
|
||||
complete_store() - marks a previous store as completed.
|
||||
Following this call, the given blocks will become loadable.
|
||||
|
||||
Exclusion:
|
||||
lock is a context manager guarding a manager's whole state subtree: the
|
||||
manager itself, every tier it composes, and any front-end state those
|
||||
tiers own. Manager methods never take the lock themselves, so a caller
|
||||
holds it across as many operations as its invariants need -- notably a
|
||||
lookup() HIT and the prepare_load() that pins the result, which must not
|
||||
be separated by another thread's eviction. The default is a no-op;
|
||||
managers that can be entered from more than one thread supply a real
|
||||
lock.
|
||||
"""
|
||||
|
||||
|
||||
@@ -268,6 +279,26 @@ class ConfigInfo:
|
||||
|
||||
|
||||
class OffloadingManager(ABC):
|
||||
@property
|
||||
def lock(self) -> AbstractContextManager[Any]:
|
||||
"""Context manager granting exclusive access to this manager's state.
|
||||
|
||||
Scope is the manager, every tier it composes, and any front-end state
|
||||
those tiers own (e.g. a tier's async lookup front end). Every caller of
|
||||
an OffloadingManager method, except shutdown(), must hold it. Manager
|
||||
methods never enter it themselves, so a caller may hold it across an
|
||||
arbitrary sequence of operations -- which is required wherever one call
|
||||
establishes a fact a later call relies on, such as a lookup() HIT
|
||||
followed by the prepare_load() that pins it.
|
||||
|
||||
The default is a no-op, for managers that are only ever entered from
|
||||
one thread. A manager that may be entered from another thread -- its
|
||||
own background thread, say -- returns a real lock instead. Callers must
|
||||
not assume either: always enter it, and never re-enter it while held,
|
||||
since a real lock need not be reentrant.
|
||||
"""
|
||||
return nullcontext()
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""Checks whether a single block is offloaded and ready to be read.
|
||||
@@ -430,6 +461,10 @@ class OffloadingManager(ABC):
|
||||
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
||||
"""Called once at the end of each scheduler step.
|
||||
|
||||
This is the last manager call of a step: every lookup(), prepare_load(),
|
||||
prepare_store() and on_request_finished() for the step has already been
|
||||
issued.
|
||||
|
||||
Managers may override this to flush deferred work accumulated
|
||||
during the step (e.g., batched promotions).
|
||||
"""
|
||||
@@ -459,7 +494,12 @@ class OffloadingManager(ABC):
|
||||
return [{}]
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Shutdown the manager and release any resources."""
|
||||
"""Shutdown the manager and release any resources.
|
||||
|
||||
The one exception to the rule that callers hold ``lock``: call this
|
||||
without it. A manager with a background thread stops that thread here
|
||||
and may need the lock to do so.
|
||||
"""
|
||||
return
|
||||
|
||||
|
||||
|
||||
@@ -11,8 +11,12 @@ Locking design
|
||||
--------------
|
||||
There is no explicit lock. Thread safety is achieved by ownership:
|
||||
|
||||
* _lookup_state and _lookup_batch are owned exclusively by the scheduler
|
||||
thread. lookup(), flush(), and cleanup() read and write them directly.
|
||||
* _lookup_state and _lookup_batch are owned by whichever thread holds the
|
||||
OffloadingManager lock. lookup(), flush(), and cleanup() read and write
|
||||
them directly. That is usually the scheduler thread, but a tier reached
|
||||
through ParentManager -- a peer lookup fanning out from another tier, say --
|
||||
can arrive on the tiering manager's tier-poll thread instead. Both hold
|
||||
the manager lock, so the accesses stay serialized.
|
||||
|
||||
* _lookup_queue is written by the scheduler (flush → put_nowait, one item
|
||||
per step) and read by the background thread (get). queue.Queue is
|
||||
|
||||
@@ -142,6 +142,12 @@ class SecondaryTierManager(ABC):
|
||||
IMPORTANT: All methods run in the Scheduler process and must be
|
||||
lightweight and non-blocking. submit_load() and submit_store() submit
|
||||
async jobs; get_finished_jobs() polls for completion.
|
||||
|
||||
Methods are called under the tiering manager's lock, so they are never
|
||||
entered concurrently -- but not always from the same thread. A tier that
|
||||
sets tolerates_off_thread_polling is also serviced from the manager's
|
||||
tier-poll thread, and any tier can be reached from there via
|
||||
ParentManager fan-out. Keep per-thread assumptions out of tier state.
|
||||
"""
|
||||
|
||||
medium: ClassVar[Medium | None] = None
|
||||
@@ -157,6 +163,16 @@ class SecondaryTierManager(ABC):
|
||||
Medium.STORAGE: CacheHitSource.DISK,
|
||||
}[cls.medium]
|
||||
|
||||
# Whether the tiering manager may also service this tier from its
|
||||
# tier-poll thread, between engine steps, rather than only from
|
||||
# on_schedule_end(). That thread calls both get_finished_jobs() and
|
||||
# serve_external_requests(); see poll_tiers(). Set it on a tier that
|
||||
# answers a counterpart the engine does not drive -- a remote peer, say --
|
||||
# whose requests would otherwise wait for a step boundary. Overriding
|
||||
# serve_external_requests() alone does not opt in: setting this asserts
|
||||
# the tier tolerates those calls off the scheduler thread.
|
||||
tolerates_off_thread_polling: ClassVar[bool] = False
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: "OffloadingSpec",
|
||||
@@ -327,9 +343,20 @@ class SecondaryTierManager(ABC):
|
||||
def serve_external_requests(self, parent: ParentManager) -> None:
|
||||
"""Process remotely-originated requests using the parent manager.
|
||||
|
||||
Called once per scheduler step, BEFORE _flush_pending_promotions().
|
||||
The parent handle is valid only for the duration of this call.
|
||||
Tiers that don't serve external requests leave this as a no-op.
|
||||
Called once per scheduler step, BEFORE _flush_pending_promotions(), and
|
||||
additionally once per tier-poll round for tiers that set
|
||||
tolerates_off_thread_polling. The _flush_pending_promotions() ordering
|
||||
holds only for the per-step call: a promotion this method initiates from
|
||||
a tier-poll round is submitted at the next on_schedule_end().
|
||||
|
||||
The parent handle is valid only for the duration of this call, and the
|
||||
caller holds the manager lock throughout it -- so a lookup() HIT taken
|
||||
here stays valid until this call pins it. An implementation must not
|
||||
release that lock partway through.
|
||||
|
||||
Tiers that don't serve external requests leave this as a no-op. A tier
|
||||
that overrides it without setting tolerates_off_thread_polling is served
|
||||
once per step only.
|
||||
"""
|
||||
return
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@ import time
|
||||
import uuid
|
||||
from collections.abc import Collection, Iterable
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any, cast
|
||||
from typing import TYPE_CHECKING, Any, ClassVar, cast
|
||||
|
||||
import msgspec
|
||||
from kvcr import (
|
||||
@@ -272,6 +272,12 @@ _RETAINED_NATIVE_BUFFERS: list[object] = []
|
||||
class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
"""Secondary tier wrapper around the KVCR KV P2P API."""
|
||||
|
||||
# Serves framework pin requests from serve_external_requests(), but only
|
||||
# once per step for now: opting in would also poll the KVCR library from
|
||||
# the tiering manager's polling thread, which it is not yet verified to
|
||||
# tolerate.
|
||||
tolerates_off_thread_polling: ClassVar[bool] = False
|
||||
|
||||
@classmethod
|
||||
@override
|
||||
def build_metric_definitions(
|
||||
|
||||
@@ -19,6 +19,7 @@ Key Design Principles:
|
||||
protecting chunks from eviction until complete_read() is called
|
||||
"""
|
||||
|
||||
import threading
|
||||
import time
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
@@ -61,6 +62,32 @@ from vllm.v1.metrics.cache_hit_source import CacheHitSource
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Default pause between tier-poll rounds; the user-facing default of the
|
||||
# ``tier_poll_interval_s`` extra-config key. A round is a non-blocking sweep, so
|
||||
# this sets how long a peer's request can sit unserved; 1 ms is three orders of
|
||||
# magnitude below the model step it hides, and matches the pause the p2p tier's
|
||||
# own drain loops and EngineCoreProc's GIL-yield sleep already use.
|
||||
DEFAULT_TIER_POLL_INTERVAL_S = 0.001
|
||||
# How long a round waits for the manager lock before giving up and retrying.
|
||||
# Bounded so the thread stays responsive to shutdown even if a caller holds the
|
||||
# lock for a long time, or leaked it.
|
||||
_TIER_POLL_LOCK_TIMEOUT_S = 1.0
|
||||
# Warn after this much continuous failure to acquire the lock. Above the 5 s
|
||||
# warning drain_jobs() emits, so a slow reset_cache() does not warn twice.
|
||||
_TIER_POLL_STALL_WARN_S = 30.0
|
||||
# Give up after rounds have failed continuously for this long. Counted in time,
|
||||
# not rounds: at the default 1 ms interval a round budget is spent in a blink,
|
||||
# so a transient fault (a peer reconnecting) would turn the thread off for
|
||||
# good. on_schedule_end() still serves the tiers, so the fallback is per-step
|
||||
# servicing rather than none.
|
||||
_TIER_POLL_GIVE_UP_AFTER_S = 30.0
|
||||
# Minimum spacing between logged tracebacks for a run of failing rounds, which
|
||||
# would otherwise log one per interval.
|
||||
_TIER_POLL_ERROR_LOG_INTERVAL_S = 1.0
|
||||
# How long shutdown() waits for the thread to finish its round.
|
||||
_TIER_POLL_JOIN_TIMEOUT_S = 5.0
|
||||
|
||||
|
||||
@dataclass
|
||||
class PendingPromotion:
|
||||
"""Accumulator for chunks awaiting submit_load() for one (tier, request)."""
|
||||
@@ -194,6 +221,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self,
|
||||
primary_tier: CPUPrimaryTierOffloadingManager,
|
||||
secondary_tiers: list[SecondaryTierManager] | None = None,
|
||||
tier_poll_interval_s: float = DEFAULT_TIER_POLL_INTERVAL_S,
|
||||
):
|
||||
"""Initialize the TieringOffloadingManager.
|
||||
|
||||
@@ -201,8 +229,16 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
primary_tier: The primary tier manager (CPU-based).
|
||||
secondary_tiers: List of secondary tier managers (e.g., Storage,
|
||||
Network). Can be None or empty list.
|
||||
tier_poll_interval_s: Pause between tier-poll rounds for
|
||||
tiers that set tolerates_off_thread_polling. Zero or negative
|
||||
disables the thread, leaving those tiers serviced once per
|
||||
engine step as before.
|
||||
|
||||
"""
|
||||
# Guards this manager and every tier it composes, the primary tier
|
||||
# included. A real lock because the tier-poll thread enters the
|
||||
# manager too; the primary tier has none of its own.
|
||||
self._lock = threading.Lock()
|
||||
self.primary_tier: CPUPrimaryTierOffloadingManager = primary_tier
|
||||
self.secondary_tiers = secondary_tiers or []
|
||||
|
||||
@@ -250,6 +286,143 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
tier: i for i, tier in enumerate(self.secondary_tiers)
|
||||
}
|
||||
|
||||
# Tiers that serve external requests, and so must also be serviced
|
||||
# between engine steps.
|
||||
self._polled_tiers: list[SecondaryTierManager] = [
|
||||
tier for tier in self.secondary_tiers if tier.tolerates_off_thread_polling
|
||||
]
|
||||
self._tier_poll_interval_s = tier_poll_interval_s
|
||||
self._tier_poll_stop = threading.Event()
|
||||
self._tier_poll_thread: threading.Thread | None = None
|
||||
self._start_tier_poll_thread()
|
||||
|
||||
@property
|
||||
@override
|
||||
def lock(self) -> threading.Lock:
|
||||
return self._lock
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Tier polling
|
||||
# ------------------------------------------------------------------
|
||||
|
||||
def _start_tier_poll_thread(self) -> None:
|
||||
"""Start the tier-poll thread, if any tier asked for one."""
|
||||
if not self._polled_tiers or self._tier_poll_interval_s <= 0:
|
||||
return
|
||||
self._tier_poll_thread = threading.Thread(
|
||||
target=self._tier_poll_loop,
|
||||
name="vllm_offload_tier_poll",
|
||||
daemon=True,
|
||||
)
|
||||
self._tier_poll_thread.start()
|
||||
logger.info(
|
||||
"KV offload tier-poll thread started for tier(s) %s, interval %.3fs",
|
||||
[tier.tier_type for tier in self._polled_tiers],
|
||||
self._tier_poll_interval_s,
|
||||
)
|
||||
|
||||
def _stop_tier_poll_thread(self) -> None:
|
||||
"""Stop and join the tier-poll thread.
|
||||
|
||||
Must run before any tier teardown: the thread drives tier transports,
|
||||
and closing one underneath it can crash the process rather than raise.
|
||||
"""
|
||||
self._tier_poll_stop.set()
|
||||
thread = self._tier_poll_thread
|
||||
if thread is None:
|
||||
return
|
||||
thread.join(timeout=_TIER_POLL_JOIN_TIMEOUT_S)
|
||||
if thread.is_alive():
|
||||
logger.error(
|
||||
"KV offload tier-poll thread did not exit within %.1fs",
|
||||
_TIER_POLL_JOIN_TIMEOUT_S,
|
||||
)
|
||||
else:
|
||||
self._tier_poll_thread = None
|
||||
|
||||
def _tier_poll_loop(self) -> None:
|
||||
stalled_since: float | None = None
|
||||
# Start of the current run of failed rounds, and when one last logged.
|
||||
failing_since: float | None = None
|
||||
last_error_log = 0.0
|
||||
while not self._tier_poll_stop.is_set():
|
||||
if not self._lock.acquire(timeout=_TIER_POLL_LOCK_TIMEOUT_S):
|
||||
now = time.monotonic()
|
||||
if stalled_since is None:
|
||||
stalled_since = now
|
||||
elif now - stalled_since > _TIER_POLL_STALL_WARN_S:
|
||||
logger.warning(
|
||||
"KV offload tier-poll thread has not acquired the "
|
||||
"manager lock for %.0fs; peer requests are only being "
|
||||
"served at engine-step boundaries.",
|
||||
now - stalled_since,
|
||||
)
|
||||
stalled_since = None
|
||||
continue
|
||||
stalled_since = None
|
||||
try:
|
||||
self.poll_tiers()
|
||||
failing_since = None
|
||||
except Exception:
|
||||
# Never let a round kill the thread silently: on_schedule_end()
|
||||
# still serves these tiers, so persistent failure degrades to
|
||||
# per-step servicing instead of stopping the tier-poll thread.
|
||||
now = time.monotonic()
|
||||
if failing_since is None:
|
||||
failing_since = now
|
||||
if now - last_error_log >= _TIER_POLL_ERROR_LOG_INTERVAL_S:
|
||||
last_error_log = now
|
||||
logger.exception("KV offload tier-poll round failed")
|
||||
finally:
|
||||
self._lock.release()
|
||||
if (
|
||||
failing_since is not None
|
||||
and time.monotonic() - failing_since >= _TIER_POLL_GIVE_UP_AFTER_S
|
||||
):
|
||||
logger.error(
|
||||
"KV offload tier-poll thread stopping after %.0fs of "
|
||||
"consecutive failures; falling back to per-step servicing.",
|
||||
_TIER_POLL_GIVE_UP_AFTER_S,
|
||||
)
|
||||
return
|
||||
# Yield unconditionally, and only after release(). Python locks are
|
||||
# not fair, so a release-then-reacquire loop could starve the
|
||||
# scheduler thread; this pause is what bounds its wait to one round.
|
||||
self._tier_poll_stop.wait(self._tier_poll_interval_s)
|
||||
|
||||
def poll_tiers(self) -> None:
|
||||
"""Advance the control plane of tiers that cannot wait for a step.
|
||||
|
||||
Polls those tiers for finished jobs and lets them serve whatever their
|
||||
counterpart has asked for -- the same two steps on_schedule_end() runs,
|
||||
narrowed to the tiers that opted in.
|
||||
|
||||
Excludes _flush_pending_promotions() and _flush_pending_cascades() by
|
||||
design: both are per-step batching points, and running them here would
|
||||
fragment batches the scheduler thread accumulated during schedule(). A
|
||||
promotion that a lookup() from this path initiates therefore waits for
|
||||
the next on_schedule_end() to be submitted.
|
||||
|
||||
The caller must hold self.lock for the whole call. serve_external_requests()
|
||||
relies on that: it establishes lookup() HITs and then pins them, and the
|
||||
two must not be separated by an eviction.
|
||||
|
||||
Serving can look keys up through the parent, and lookup() runs the
|
||||
step's once-per-step poll. From here that poll would reach every tier
|
||||
off-thread, re-enter the serving tier's get_finished_jobs() from inside
|
||||
its own serve, and leave the gate set so the next step skips its poll.
|
||||
The round has just polled its tiers itself, so it holds the gate shut
|
||||
for its duration and then puts it back untouched.
|
||||
"""
|
||||
gate = self._processed_jobs_this_step
|
||||
self._processed_jobs_this_step = True
|
||||
try:
|
||||
self._process_finished_jobs(self._polled_tiers)
|
||||
for tier in self._polled_tiers:
|
||||
tier.serve_external_requests(self._tier_parents[tier])
|
||||
finally:
|
||||
self._processed_jobs_this_step = gate
|
||||
|
||||
@property
|
||||
def _transfer_jobs(self) -> dict[JobId, JobMetadata]:
|
||||
return self._jobs
|
||||
@@ -321,8 +494,11 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
False,
|
||||
)
|
||||
|
||||
def _process_finished_jobs(self):
|
||||
"""Unconditionally poll all secondary tiers for completed jobs.
|
||||
def _process_finished_jobs(
|
||||
self,
|
||||
tiers: Sequence[SecondaryTierManager] | None = None,
|
||||
):
|
||||
"""Unconditionally poll secondary tiers for completed jobs.
|
||||
|
||||
This method:
|
||||
1. Calls get_finished_jobs() on each secondary tier
|
||||
@@ -330,8 +506,18 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
to decrement ref_cnt
|
||||
3. For completed loads (secondary→primary): calls primary.complete_write()
|
||||
to make chunks available
|
||||
|
||||
Args:
|
||||
tiers: Tiers to poll. Defaults to every secondary tier. Pass a
|
||||
subset to poll only the tiers a caller is responsible for, so
|
||||
a tier that expects to be polled on the scheduler thread is
|
||||
not dragged elsewhere.
|
||||
|
||||
"""
|
||||
for i, tier in enumerate(self.secondary_tiers):
|
||||
if tiers is None:
|
||||
tiers = self.secondary_tiers
|
||||
for tier in tiers:
|
||||
i = self._tier_index[tier]
|
||||
for completed_job in tier.get_finished_jobs():
|
||||
job_id = completed_job.job_id
|
||||
job_metadata = self._pop_job(job_id)
|
||||
@@ -622,9 +808,9 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
) -> PrepareStoreOutput | None:
|
||||
"""Prepare chunks to be stored from GPU to primary tier.
|
||||
|
||||
CRITICAL: This method calls _maybe_process_finished_jobs() FIRST to ensure
|
||||
that any completed async transfers have their ref_cnt decremented
|
||||
before the primary tier makes eviction decisions.
|
||||
CRITICAL: This method polls for finished jobs FIRST to ensure that any
|
||||
completed async transfers have their ref_cnt decremented before the
|
||||
primary tier makes eviction decisions.
|
||||
|
||||
For request-level tiers, chunks already present in the primary tier
|
||||
are immediately cascaded via submit_store().
|
||||
@@ -648,7 +834,12 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# not-yet-ready chunk's ref_cnt from -1 to 0 via complete_write(),
|
||||
# making it evictable for the first time.
|
||||
# Both must be accounted for before the eviction decision below.
|
||||
self._maybe_process_finished_jobs()
|
||||
# Unconditional, not _maybe_process_finished_jobs(): eviction must see
|
||||
# the freshest completions, and on_schedule_end now runs at the very end
|
||||
# of the step, so the once-per-step gate would otherwise still be set by
|
||||
# this step's first lookup() and skip the poll.
|
||||
self._processed_jobs_this_step = True
|
||||
self._process_finished_jobs()
|
||||
|
||||
# Step 2: Store to primary tier (new chunks only).
|
||||
# Cascading of these newly-stored chunks to ALL secondary tiers
|
||||
@@ -884,8 +1075,8 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
"""End-of-schedule hook: process finished jobs, flush deferred
|
||||
promotions, and reset the per-step gate.
|
||||
|
||||
Called once per scheduler step from
|
||||
OffloadingConnectorScheduler.build_connector_meta().
|
||||
Called once per scheduler step, as the last manager call of the step,
|
||||
from OffloadingConnectorScheduler.build_connector_meta().
|
||||
"""
|
||||
# Catch-all poll: guarantees jobs are processed even on steps where
|
||||
# lookup()/prepare_store() were never called (e.g. no requests
|
||||
@@ -900,6 +1091,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
self._flush_pending_promotions()
|
||||
# Keys parked by this step's prepare_store cannot have completed yet, so
|
||||
# they are simply re-parked here and retried on the next step. That is
|
||||
# safe: _maybe_finalize_request() holds off while pending_cascade_keys is
|
||||
# non-empty, so nothing finalizes early and the list still drains.
|
||||
self._flush_pending_cascades()
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_schedule_end(context)
|
||||
@@ -1033,9 +1228,40 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
def shutdown(self) -> None:
|
||||
"""Shut down secondary tiers before releasing primary resources.
|
||||
|
||||
Stops the tier-poll thread first: it drives tier transports, and
|
||||
tearing one down underneath it (closing a socket, destroying a ZMQ
|
||||
context) can take the process down rather than raise. If the thread
|
||||
will not exit, hold the manager lock across tier teardown so it cannot
|
||||
be mid-round, and raise if even that is unavailable -- tearing a tier
|
||||
down under a live round risks a crash, and skipping it silently would
|
||||
leak the primary's shared memory region.
|
||||
|
||||
Unlike every other manager method, call this WITHOUT holding
|
||||
self.lock: it takes the lock itself, which is not reentrant, and the
|
||||
tier-poll thread needs the lock free to notice the stop request
|
||||
and exit. A caller that does hold it only gets the RuntimeError above,
|
||||
after waiting out the timeout.
|
||||
|
||||
Every secondary tier is given a shutdown attempt. If any shutdown
|
||||
fails, preserve the primary mmap because a failed tier may still use it.
|
||||
"""
|
||||
self._stop_tier_poll_thread()
|
||||
if self._tier_poll_thread is not None:
|
||||
if not self._lock.acquire(timeout=_TIER_POLL_JOIN_TIMEOUT_S):
|
||||
raise RuntimeError(
|
||||
"KV offload tier-poll thread is still running and the "
|
||||
"manager lock is unavailable; secondary tiers were not "
|
||||
"shut down. shutdown() must be called without holding "
|
||||
"manager.lock."
|
||||
)
|
||||
try:
|
||||
self._shutdown_tiers()
|
||||
finally:
|
||||
self._lock.release()
|
||||
return
|
||||
self._shutdown_tiers()
|
||||
|
||||
def _shutdown_tiers(self) -> None:
|
||||
shutdown_error: Exception | None = None
|
||||
for tier_idx, tier in enumerate(self.secondary_tiers):
|
||||
try:
|
||||
|
||||
@@ -20,8 +20,11 @@ Architecture
|
||||
├── mark_dead() — signal that the peer is gone
|
||||
└── close() — tear down the connection
|
||||
|
||||
Threading model: all I/O is driven by the caller invoking poll().
|
||||
No background threads. poll() must be called periodically to:
|
||||
Threading model: all I/O is driven by the caller invoking poll(). No background
|
||||
threads of its own. More than one caller thread may drive poll() over a
|
||||
transport's lifetime, but never concurrently -- callers serialize themselves
|
||||
(the tiering manager does so with its lock), because sockets here are not
|
||||
thread-safe. poll() must be called periodically to:
|
||||
- receive messages (buffered per-connection)
|
||||
- accept new inbound peers
|
||||
- detect disconnections
|
||||
|
||||
@@ -66,7 +66,11 @@ Implementor contracts
|
||||
- close() releases all resources (memory registrations, handles).
|
||||
After close(), no other methods may be called.
|
||||
|
||||
Threading model: no background threads. All I/O driven by poll().
|
||||
Threading model: no background Python threads. All I/O driven by poll(). More
|
||||
than one caller thread may drive a transport over its lifetime, but never
|
||||
concurrently -- callers serialize themselves (the tiering manager does so with
|
||||
its lock). Note that serialization alone is not obviously sufficient for a
|
||||
backend that binds state to the thread that created it; see NixlTransport.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -1,6 +1,37 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL."""
|
||||
"""NixlTransport: Data-plane transport for RDMA-based KV block transfers via NIXL.
|
||||
|
||||
Threading: the agent is reached from the scheduler thread and, when the tiering
|
||||
manager runs one, from its tier-poll thread. The manager's lock serializes
|
||||
them strictly -- one Python thread inside any agent call at a time, with a full
|
||||
memory barrier between.
|
||||
|
||||
That lock is the only protection, not a belt over NIXL's own. nixl_agent picks
|
||||
NIXL_THREAD_SYNC_STRICT only when ``enable_listen`` is set, which this transport
|
||||
does not set, so the agent runs with NIXL_THREAD_SYNC_NONE -- under which NIXL's
|
||||
internal agent lock compiles down to no-ops, leaving every agent entry point
|
||||
unguarded. So the manager lock must cover all of them, and the lifetime of every
|
||||
handle passed between calls. The exception is peer registration moved onto a
|
||||
worker thread, as NixlConnector's handshake executor does: it runs outside the
|
||||
lock and relies, like that connector, on NIXL tolerating it beside transfers.
|
||||
|
||||
Serialized use from a second thread is supported (checked against NIXL 1.4.1
|
||||
sources): NIXL never creates its UCX worker in UCS_THREAD_MODE_SINGLE, the only
|
||||
creator-thread-affine mode. The weakest mode it ever requests is
|
||||
UCS_THREAD_MODE_SERIALIZED, documented as "multiple threads can access, but only
|
||||
one at a time", and because ``enable_prog_thread`` defaults on here it actually
|
||||
requests UCS_THREAD_MODE_MULTI and fails construction if the UCX build cannot
|
||||
provide it.
|
||||
|
||||
Two caveats. NIXL binds a shared UCX worker per calling thread
|
||||
(``tlsSharedWorkerMap``), which is harmless only while there is exactly one
|
||||
shared worker -- true here, but it would stop being true if a ``num_workers``
|
||||
UCX backend param were ever set. And NIXL 1.4.1 does accept an explicit
|
||||
``sync_mode`` (including NIXL_THREAD_SYNC_RW), which would make its own locking
|
||||
real; that parameter does not exist before 1.x, so passing it would need a
|
||||
version gate, the way ``capture_telemetry`` does.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
|
||||
@@ -206,13 +206,18 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
blocks from the peer) and server-role (serving blocks to the peer)
|
||||
over the same control connection.
|
||||
|
||||
Single-threaded: every public method runs on the scheduler thread, and
|
||||
the engine drives polling via ``get_finished_jobs()`` once per step.
|
||||
Never entered concurrently: every public method runs under the tiering
|
||||
manager's ``lock``, held either by the scheduler thread for the length of
|
||||
a step or by the tier-poll thread for the length of one round. Peers are
|
||||
not driven by the engine, so this tier sets ``tolerates_off_thread_polling``
|
||||
to be serviced from that thread too; without it a peer's lookup or fetch
|
||||
waits for a step boundary, which on a saturated rank can be seconds.
|
||||
``has_pending_work()`` keeps the engine ticking so the control transport
|
||||
and existing sessions are polled even when no requests are scheduled.
|
||||
"""
|
||||
|
||||
cache_hit_source: ClassVar[CacheHitSource] = CacheHitSource.P2P
|
||||
tolerates_off_thread_polling: ClassVar[bool] = True
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
@@ -614,9 +619,10 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
|
||||
@override
|
||||
def get_finished_jobs(self) -> Iterable[JobResult]:
|
||||
# Drive one polling sweep on the scheduler thread, then hand off
|
||||
# whatever has accumulated. The engine calls this once per step
|
||||
# (and keeps stepping while has_pending_work() is True).
|
||||
# Drive one polling sweep, then hand off whatever has accumulated. The
|
||||
# engine calls this once per step (and keeps stepping while
|
||||
# has_pending_work() is True); the tier-poll thread calls it between
|
||||
# steps, so a peer is not left waiting for a step boundary.
|
||||
self._poll_once()
|
||||
result = self._finished_jobs
|
||||
self._finished_jobs = []
|
||||
@@ -930,7 +936,8 @@ class P2PSecondaryTierManager(SecondaryTierManager):
|
||||
|
||||
Drains the control transport, polls every session, accumulates
|
||||
their results into ``_finished_jobs``, and reaps any dead sessions.
|
||||
Runs on the scheduler thread.
|
||||
Runs under the tiering manager's lock, on the scheduler thread or on
|
||||
the tier-poll thread.
|
||||
"""
|
||||
new_connections = self._control.poll()
|
||||
if new_connections:
|
||||
|
||||
@@ -602,10 +602,13 @@ class ServerRole:
|
||||
"""Pin primary slots for HIT keys and park them as the lookup's
|
||||
round supply via ``add_stored_blocks``.
|
||||
|
||||
Caller has already confirmed every key is HIT (single-threaded
|
||||
scheduler ⇒ no eviction race), so the JobMetadata returned by
|
||||
``parent.create_store_job`` carries parallel ``keys``/``block_ids``
|
||||
of length ``len(keys)``.
|
||||
Caller has already confirmed every key is HIT, so the JobMetadata
|
||||
returned by ``parent.create_store_job`` carries parallel
|
||||
``keys``/``block_ids`` of length ``len(keys)``. Nothing can evict
|
||||
between that confirmation and the pin because the whole
|
||||
``serve_external_requests`` call runs inside one hold of the tiering
|
||||
manager's lock -- which is why that call must never release it
|
||||
partway through.
|
||||
"""
|
||||
meta = parent.create_store_job(keys, lookup.ctx)
|
||||
self.add_stored_blocks(
|
||||
|
||||
@@ -76,6 +76,7 @@ from vllm.v1.kv_offload.tiering.base import (
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.factory import SecondaryTierFactory
|
||||
from vllm.v1.kv_offload.tiering.manager import (
|
||||
DEFAULT_TIER_POLL_INTERVAL_S,
|
||||
CPUPrimaryTierOffloadingManager,
|
||||
TieringOffloadingManager,
|
||||
)
|
||||
@@ -285,6 +286,13 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
if not isinstance(self.secondary_tier_configs, list):
|
||||
raise ValueError("secondary_tiers must be a list of tier configurations")
|
||||
|
||||
# Pause between tier-poll rounds for tiers that need servicing
|
||||
# between engine steps. Zero or negative disables the thread, which
|
||||
# restores per-step servicing.
|
||||
self.tier_poll_interval_s = float(
|
||||
self.extra_config.get("tier_poll_interval_s", DEFAULT_TIER_POLL_INTERVAL_S)
|
||||
)
|
||||
|
||||
# Backpressure config is merged field-by-field in priority order
|
||||
# (highest first):
|
||||
# 1. Per-tier ``backpressure`` dict in the tier config
|
||||
@@ -385,6 +393,7 @@ class TieringOffloadingSpec(CPUOffloadingSpec):
|
||||
tiering_manager = TieringOffloadingManager(
|
||||
primary_tier=primary_tier,
|
||||
secondary_tiers=secondary_tiers,
|
||||
tier_poll_interval_s=self.tier_poll_interval_s,
|
||||
)
|
||||
self._manager = tiering_manager
|
||||
except Exception:
|
||||
|
||||
Reference in New Issue
Block a user