[KV Offload] Expose OffloadingManager.lock()/unlock() and service the tiering control plane off the scheduler thread (#58168)

Signed-off-by: Liran Schour <lirans@il.ibm.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
liranschour
2026-10-15 08:50:56 +03:00
committed by GitHub
co-authored by Kevin H. Luu
parent c54dca96bd
commit c5f8adc0b1
15 changed files with 1148 additions and 38 deletions
@@ -2,7 +2,8 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import time
from collections import Counter
from collections.abc import Iterable, Sequence
from collections.abc import Iterable, Iterator, Sequence
from contextlib import ExitStack, contextmanager
from dataclasses import dataclass, field, replace
from itertools import chain, islice
from typing import Any, NamedTuple
@@ -640,6 +641,71 @@ class OffloadingConnectorScheduler:
self._events_tracker = OffloadingEventsTracker(spec.kv_events_config)
# Holds the manager lock for the current schedule(), or None between steps.
# Read and written only by the scheduler thread, so it needs no
# synchronization of its own. Declared on the class so an instance built
# without __init__ still reads None rather than raising.
_step_region: ExitStack | None = None
# ------------------------------------------------------------------
# Manager exclusion
#
# The manager is entered from the scheduler thread and, for managers that
# run one, from a tier-poll thread. Manager methods never take their
# own lock, so the exclusion boundary lives here. manager.lock may be a
# no-op for a manager that needs no exclusion; it is entered regardless.
#
# schedule() holds one region for the whole step: opened by the first
# manager-touching hook and closed at the end of build_connector_meta.
# That span is required, not tidiness -- get_num_new_matched_tokens
# establishes lookup() HITs that update_state_after_alloc later pins with
# prepare_load(), and prepare_store() evicts after both. Releasing between
# them would let another thread evict a block the step already treated as
# present.
#
# Hooks outside schedule() use a short region instead, so the manager is
# free while the engine runs the model.
#
# The step region outlives any single call, so it cannot be a with-block;
# an ExitStack carries the entered lock from open to close instead.
# ------------------------------------------------------------------
def _acquire_step(self) -> None:
"""Open the step-scoped region. Idempotent within a step."""
if self._step_region is not None:
return
region = ExitStack()
region.enter_context(self.manager.lock)
self._step_region = region
def _release_step(self) -> None:
"""Close the step-scoped region, letting other threads in."""
region = self._step_region
if region is None:
return
self._step_region = None
region.close()
@contextmanager
def _manager_locked(self) -> Iterator[None]:
"""Hold the manager for the duration of one hook outside schedule().
None of these hooks can legitimately run inside the step-scoped region:
on_new_request comes from add_request, has_pending_push_work from the
engine's liveness check before schedule(), the rest from output
processing or a client RPC. So finding the region open here means a
previous schedule() raised between _acquire_step() and the release in
build_connector_meta. Recover instead of deadlocking.
"""
if self._step_region is not None:
logger.error(
"Offloading manager lock was left held by a previous "
"schedule(); releasing it before continuing."
)
self._release_step()
with self.manager.lock:
yield
def _maybe_observe_lookup_async_delay(
self, req_status: RequestOffloadState
) -> None:
@@ -1031,7 +1097,8 @@ class OffloadingConnectorScheduler:
def on_new_request(self, request: Request) -> None:
"""Called when a new request is added to the scheduler."""
req_context = _create_req_context(request)
offloading_context = self.manager.on_new_request(req_context)
with self._manager_locked():
offloading_context = self.manager.on_new_request(req_context)
req_status = RequestOffloadState(
config=self.config,
req=request,
@@ -1067,6 +1134,10 @@ class OffloadingConnectorScheduler:
(between scheduler steps).
"""
# First manager-touching hook of a step: opens the step-scoped region,
# closed at the end of build_connector_meta.
self._acquire_step()
req_status = self._req_status[request.request_id]
for group_state in req_status.group_states:
group_state.block_ids.clear()
@@ -1161,6 +1232,10 @@ class OffloadingConnectorScheduler:
if num_external_tokens == 0:
return
# prepare_load() below pins keys that get_num_new_matched_tokens()
# already saw as HIT, so both must sit inside the same region.
self._acquire_step()
req_status = self._req_status[request.request_id]
num_locally_computed_tokens = req_status.num_locally_computed_tokens
@@ -1853,13 +1928,24 @@ class OffloadingConnectorScheduler:
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
# A step that scheduled nothing reaches build_connector_meta without
# any earlier manager touch, so open the region here too.
self._acquire_step()
try:
return self._build_connector_meta(scheduler_output)
finally:
# The step is over: hand the manager back.
self._release_step()
def _build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
self._update_req_states(scheduler_output)
schedule_end_context = ScheduleEndContext(
new_req_ids=[req.req_id for req in scheduler_output.scheduled_new_reqs],
preempted_req_ids=scheduler_output.preempted_req_ids or (),
)
self.manager.on_schedule_end(schedule_end_context)
# Flush jobs for preempted requests.
for req_id in scheduler_output.preempted_req_ids or ():
@@ -1905,6 +1991,12 @@ class OffloadingConnectorScheduler:
self._current_batch_load_jobs = {}
self._current_batch_jobs_to_flush = set()
self._current_batch_allocated_block_ids = set()
# Last manager call of the step, after every prepare_store and
# on_request_finished above. on_schedule_end is documented as running at
# the end of the step, and callers rely on that to bound the step's
# exclusive access to the manager.
self.manager.on_schedule_end(schedule_end_context)
return meta
def has_pending_push_work(self) -> bool:
@@ -1912,8 +2004,17 @@ class OffloadingConnectorScheduler:
While True, build_connector_meta() and update_connector_output()
continue to be called even when no requests are scheduled.
Must use a short region, never the step-scoped one. The engine asks this
before schedule() and returns early when the answer is False, so a
step-scoped acquire here would never reach build_connector_meta's
release and would hold the manager for good -- which is precisely the
state an idle engine settles into.
"""
return bool(self._jobs) or self.manager.has_pending_work()
if self._jobs:
return True
with self._manager_locked():
return self.manager.has_pending_work()
def update_connector_output(self, connector_output: KVConnectorOutput):
"""Update KVConnector state from worker-side connectors output.
@@ -1923,6 +2024,10 @@ class OffloadingConnectorScheduler:
connectors output.
"""
with self._manager_locked():
self._update_connector_output(connector_output)
def _update_connector_output(self, connector_output: KVConnectorOutput):
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, OffloadingWorkerMetadata):
assert meta is None
@@ -2001,7 +2106,8 @@ class OffloadingConnectorScheduler:
stats = self._connector_stats
self._connector_stats = OffloadingConnectorStats()
manager_stats = self.manager.get_stats()
with self._manager_locked():
manager_stats = self.manager.get_stats()
if manager_stats is not None:
if stats is None:
stats = manager_stats
@@ -2038,8 +2144,9 @@ class OffloadingConnectorScheduler:
# Untracked request (offloading never started): no in-flight jobs,
# nothing was deferred, so finalize immediately.
req_context = _create_req_context(request)
self.manager.on_new_request(req_context)
self.manager.on_request_finished(req_context)
with self._manager_locked():
self.manager.on_new_request(req_context)
self.manager.on_request_finished(req_context)
return False, None
self._maybe_observe_lookup_async_delay(req_status)
@@ -2071,7 +2178,16 @@ class OffloadingConnectorScheduler:
the underlying :class:`OffloadingEvent` stream.
"""
yield from self._events_tracker.take_events(self.manager.take_events())
# Drain the manager eagerly, inside the region. manager.take_events() is
# itself a generator whose backing list is cleared once exhausted, so
# iterating it lazily would both hold the manager for as long as the
# consumer takes and let a concurrent writer append events that the
# clear then discards unpublished.
with self._manager_locked():
manager_events = list(self.manager.take_events())
# Still a generator, so callers testing the result for truthiness keep
# seeing a non-empty object the way they do today.
yield from self._events_tracker.take_events(manager_events)
def reset_cache(self) -> None:
"""Reset the offloading manager cache, evicting all stored chunks."""
@@ -2080,6 +2196,13 @@ class OffloadingConnectorScheduler:
assert not self._current_batch_jobs_to_flush
assert not self._current_batch_allocated_block_ids
# Held for the whole reset. manager.reset_cache() drains tiers by
# polling them itself, so a tier-poll thread locked out here loses
# nothing; it simply skips rounds until the reset completes.
with self._manager_locked():
self._reset_cache()
def _reset_cache(self) -> None:
# Flush all in-flight jobs
self._current_batch_jobs_to_flush.update(self._jobs.keys())