KV-Cache multi-tier offloading async batched lookup (#44193)

Signed-off-by: Effi Ofer <effi.ofer@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Effi Ofer
2026-06-10 14:59:30 +00:00
committed by GitHub
co-authored by Or Ozeri
parent 3cc9fecd58
commit af65e08fc5
6 changed files with 518 additions and 31 deletions
+231
View File
@@ -0,0 +1,231 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
AsyncLookupManager: per-tier async lookup manager for secondary tier
existence checks.
Each secondary tier that wants non-blocking lookups composes its own
AsyncLookupManager instance internally. The manager maintains lookup
state and uses a background thread to execute batch_lookup() calls.
Locking design
--------------
There is no explicit lock. Thread safety is achieved by ownership:
* _lookup_state and _lookup_batch are owned exclusively by the scheduler
thread. lookup(), flush(), and cleanup() read and write them directly.
* _lookup_queue is written by the scheduler (flush → put_nowait, one item
per step) and read by the background thread (get). queue.Queue is
thread-safe.
* _pending_results is written by the background thread (put) and read by
the scheduler (get_nowait inside drain_results). queue.SimpleQueue is
thread-safe by design.
lookup() accumulates new keys in _lookup_batch without touching the queue.
flush() is called once per step from the tier's on_schedule_end(), posting
the entire batch as a single queue item so the background thread sees one
batch per step.
drain_results() is called before any lookup() calls in the same step, so
lookup() is a pure OrderedDict operation.
"""
import queue
import threading
from abc import ABC, abstractmethod
from collections.abc import Iterable
from dataclasses import dataclass, field
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
logger = init_logger(__name__)
@dataclass(slots=True)
class LookupState:
result: bool | None = None # True (found), False (not found), None
request_ids: set[str] = field(default_factory=set) # requests asking for the lookup
class AsyncLookupManager(ABC):
"""
Per-tier async lookup manager for secondary tier existence checks.
Each secondary tier that wants non-blocking lookups composes its own
AsyncLookupManager instance internally. The manager maintains lookup
state (cache, queue) and uses a background thread to execute the actual
batch_lookup() calls.
Subclasses implement only batch_lookup() — all queue management,
state tracking, and result delivery is provided by this base class.
The owning tier delegates its lookup(), on_schedule_end(), and
on_request_finished() to this manager:
- lookup() → drain_results() + lookup state check
- on_schedule_end() → flush()
- on_request_finished() → cleanup()
"""
def __init__(
self,
tier_type: str,
) -> None:
self._tier_type = tier_type
# key → LookupState; scheduler-owned, no lock needed.
self._lookup_state: dict[OffloadKey, LookupState] = {}
# req_id → keys looked up by that request (reverse index for cleanup).
self._req_keys: dict[str, set[OffloadKey]] = {}
# Accumulates (key, req_context) pairs during lookup() calls.
# Flushed as one queue item per step by flush().
self._lookup_batch: list[tuple[OffloadKey, ReqContext]] = []
# Scheduler → worker: one full step's batch per item.
# None is used as a shutdown sentinel.
self._lookup_queue: queue.SimpleQueue[
list[tuple[OffloadKey, ReqContext]] | None
] = queue.SimpleQueue()
# Worker → scheduler: completed result batches.
# Each item is a list of (key, found) pairs.
# SimpleQueue is explicitly thread-safe for one writer / one reader.
self._pending_results: queue.SimpleQueue[list[tuple[OffloadKey, bool]]] = (
queue.SimpleQueue()
)
self._need_to_drain: bool = False
self._thread = threading.Thread(
target=self._worker,
name=f"vllm_offloading_lookup_{tier_type}",
daemon=True,
)
self._thread.start()
@abstractmethod
def batch_lookup(
self, keys: list[OffloadKey], req_context: ReqContext
) -> Iterable[bool]:
"""
Check whether a batch of blocks exist in this tier.
Called from the worker thread — must be synchronous and must not
touch the primary tier or scheduler state.
Returns a list parallel to keys: True if present, False if not.
"""
...
# ------------------------------------------------------------------
# Scheduler-thread API
# ------------------------------------------------------------------
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
"""
Non-blocking lookup called from the scheduler thread.
Returns:
True — block is present in this tier.
False — block is not present in this tier.
None — result not yet available; retry next step.
"""
if self._need_to_drain:
self.drain_results()
self._need_to_drain = False
req_id = req_context.req_id
state = self._lookup_state.get(key)
if state is None:
state = LookupState()
self._lookup_state[key] = state
self._lookup_batch.append((key, req_context))
state.request_ids.add(req_id)
self._req_keys.setdefault(req_id, set()).add(key)
return state.result
def flush(self) -> None:
"""Post this step's accumulated keys to the worker thread.
Called once per step from on_schedule_end() after all lookup() calls
are done. The worker receives the full batch and processes it during
the model-execution window, maximising time available before the next
step's drain_results(). Safe to call with an empty batch (no-op).
"""
self._need_to_drain = True
if self._lookup_batch:
self._lookup_queue.put(self._lookup_batch)
self._lookup_batch = []
def drain_results(self) -> None:
"""Apply pending worker results to _lookup_state.
Called from lookup() before checking state.
"""
while True:
try:
batch = self._pending_results.get_nowait()
except queue.Empty:
break
for key, result in batch:
state = self._lookup_state.get(key)
if state is not None:
state.result = result
def cleanup(self, req_id: str) -> None:
"""Remove entries no longer needed by any active request.
Called from the tier's on_request_finished(). Uses the reverse
index to visit only keys associated with this request.
"""
for key in self._req_keys.pop(req_id, ()):
state = self._lookup_state[key]
state.request_ids.discard(req_id)
if not state.request_ids:
del self._lookup_state[key]
def shutdown(self) -> None:
"""Stop the worker thread."""
self._lookup_queue.put(None) # unblock _worker from _lookup_queue.get()
self._thread.join()
# ------------------------------------------------------------------
# Internal helpers
# ------------------------------------------------------------------
def _worker(self) -> None:
while True:
pending = self._lookup_queue.get()
if pending is None:
break
# Group by req_id.
batches: dict[str, tuple[ReqContext, list[OffloadKey]]] = {}
for key, req_context in pending:
req_id = req_context.req_id
if req_id not in batches:
batches[req_id] = (req_context, [])
batches[req_id][1].append(key)
if not batches:
continue
results: list[tuple[OffloadKey, bool]] = []
for req_context, keys in batches.values():
try:
hits = self.batch_lookup(keys, req_context)
except Exception as exc:
logger.warning(
"batch_lookup failed on tier %s for %d keys: %s",
self._tier_type,
len(keys),
exc,
)
hits = (False for _ in keys)
for key, hit in zip(keys, hits):
results.append((key, hit))
# Post the entire batch as one item — no lock needed.
if results:
self._pending_results.put(results)
+33 -6
View File
@@ -26,6 +26,7 @@ from typing_extensions import override
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
@@ -41,6 +42,23 @@ if TYPE_CHECKING:
logger = init_logger(__name__)
class FsAsyncLookupManager(AsyncLookupManager):
"""Async lookup manager for FileSystemTierManager."""
def __init__(
self,
tier: "FileSystemTierManager",
tier_type: str,
) -> None:
super().__init__(tier_type=tier_type)
self._tier = tier
def batch_lookup(
self, keys: list[OffloadKey], req_context: ReqContext
) -> Iterable[bool]:
return (os.path.exists(self._tier.file_mapper.get_file_name(k)) for k in keys)
class FileSystemTierManager(SecondaryTierManager):
"""
Pure-Python disk-backed secondary tier.
@@ -111,15 +129,15 @@ class FileSystemTierManager(SecondaryTierManager):
thread_name_prefix="vllm_kv_py_fs",
)
self._lookup_manager = FsAsyncLookupManager(tier=self, tier_type=self.tier_type)
@override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
return RequestOffloadingContext()
@override
def lookup(
self, key: OffloadKey, req_context: ReqContext | None = None
) -> bool | None:
return os.path.exists(self.file_mapper.get_file_name(key))
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
return self._lookup_manager.lookup(key, req_context)
@override
def submit_store(self, job_metadata: JobMetadata) -> None:
@@ -159,12 +177,21 @@ class FileSystemTierManager(SecondaryTierManager):
for job_id, success in self._pool.get_finished()
)
@override
def on_request_finished(self, req_context: ReqContext) -> None:
self._lookup_manager.cleanup(req_context.req_id)
@override
def on_schedule_end(self) -> None:
self._lookup_manager.flush()
@override
def shutdown(self) -> None:
"""
Release resources held by this tier.
Shuts down the thread pool, clearing pending tasks and waiting for
active threads to complete.
Shuts down the lookup manager and the thread pool,
clearing pending tasks and waiting for active threads to complete.
"""
self._lookup_manager.shutdown()
self._pool.shutdown(wait=True)
+44 -5
View File
@@ -11,6 +11,7 @@ from vllm.distributed.nixl_utils import nixl_agent_config
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
@@ -49,6 +50,37 @@ class TransferEntry(NamedTuple):
obj_handle: "nixl_prepped_dlist_handle"
class ObjAsyncLookupManager(AsyncLookupManager):
"""Async lookup manager for ObjectStoreSecondaryTierManager.
Batches existence probes into a single query_memory() call so the
background thread issues one round-trip per step instead of one per key.
"""
def __init__(
self,
tier: "ObjectStoreSecondaryTierManager",
tier_type: str,
) -> None:
super().__init__(tier_type=tier_type)
self._tier = tier
def batch_lookup(
self, keys: list[OffloadKey], req_context: ReqContext
) -> Iterable[bool]:
descriptors = [
(
_PROBE_ADDR,
_PROBE_LEN,
_PROBE_DEV_ID,
self._tier._file_mapper.get_file_name(k),
)
for k in keys
]
results = self._tier._agent.query_memory(descriptors, "OBJ", "OBJ")
return (r is not None for r in results)
class ObjectStoreSecondaryTierManager(SecondaryTierManager):
"""Secondary tier that offloads KV cache blocks to an S3-compatible store.
@@ -99,6 +131,10 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", all_blocks, "DRAM")
)
self._lookup_manager = ObjAsyncLookupManager(
tier=self, tier_type=self.tier_type
)
def _probe_connectivity(self) -> None:
"""Verify object store connectivity at startup via a NIXL lookup probe.
@@ -179,11 +215,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
self._transfers[job_id] = TransferEntry(xfer_handle, files_desc, obj_handle)
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
try:
return self._exists(self._file_mapper.get_file_name(key))
except Exception as e:
logger.warning("lookup failed for key %s: %s", key, e)
return False
return self._lookup_manager.lookup(key, req_context)
def submit_store(self, job_metadata: JobMetadata) -> None:
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
@@ -197,6 +229,12 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_READ
)
def on_request_finished(self, req_context: ReqContext) -> None:
self._lookup_manager.cleanup(req_context.req_id)
def on_schedule_end(self) -> None:
self._lookup_manager.flush()
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
return RequestOffloadingContext()
@@ -226,6 +264,7 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
return results
def shutdown(self) -> None:
self._lookup_manager.shutdown()
for job_id, entry in self._transfers.items():
try:
self._agent.release_xfer_handle(entry.xfer_handle)