Signed-off-by: Bill Nell <bnell@redhat.com> Signed-off-by: bnellnm <49004751+bnellnm@users.noreply.github.com> Co-authored-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
462 lines
18 KiB
Python
462 lines
18 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""Object store secondary tier implementation."""
|
|
|
|
import ctypes
|
|
import time
|
|
from collections.abc import Iterable
|
|
from typing import TYPE_CHECKING, ClassVar, NamedTuple
|
|
|
|
import numpy as np
|
|
|
|
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
|
|
from vllm.distributed.nixl_utils import nixl_agent_config
|
|
from vllm.logger import init_logger
|
|
from vllm.v1.kv_offload.base import (
|
|
Locality,
|
|
LookupResult,
|
|
Medium,
|
|
OffloadingEvent,
|
|
OffloadKey,
|
|
ReqContext,
|
|
)
|
|
from vllm.v1.kv_offload.file_mapper import FileMapper
|
|
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
|
from vllm.v1.kv_offload.tiering.backpressure import BackpressureDetector
|
|
from vllm.v1.kv_offload.tiering.base import (
|
|
JobId,
|
|
JobResult,
|
|
RequestOffloadingContext,
|
|
ScheduleEndContext,
|
|
SecondaryTierManager,
|
|
TransferJob,
|
|
)
|
|
from vllm.v1.kv_offload.tiering.obj.config import ObjStoreConfig
|
|
|
|
if TYPE_CHECKING:
|
|
from nixl._api import nixl_prepped_dlist_handle, nixl_xfer_handle
|
|
|
|
from vllm.v1.kv_offload.base import OffloadingSpec
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
NIXL_WRITE = "WRITE"
|
|
NIXL_READ = "READ"
|
|
NIXL_PROC = "PROC"
|
|
NIXL_DONE = "DONE"
|
|
|
|
# Device ID for CPU DRAM descriptors. DRAM is not a multi-device resource so
|
|
# the device ID is always 0.
|
|
NIXL_DEV_ID: int = 0
|
|
|
|
# Fields for NIXL OBJ descriptors: (addr, len, dev_id, obj_key).
|
|
# For existence probes addr and len are placeholders — no data is read.
|
|
# dev_id=0 is reserved for probes; transfers start from 1.
|
|
_PROBE_ADDR: int = 0
|
|
_PROBE_LEN: int = 1
|
|
_PROBE_DEV_ID: int = 0
|
|
|
|
|
|
class TransferEntry(NamedTuple):
|
|
xfer_handle: "nixl_xfer_handle"
|
|
files_desc: object
|
|
obj_handle: "nixl_prepped_dlist_handle"
|
|
|
|
|
|
class ObjAsyncLookupManager(AsyncLookupManager):
|
|
"""Async lookup manager for ObjectStoreSecondaryTierManager.
|
|
|
|
Batches existence probes into a single query_memory() call so the
|
|
background thread issues one round-trip per step instead of one per key.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
tier: "ObjectStoreSecondaryTierManager",
|
|
tier_type: str,
|
|
) -> None:
|
|
super().__init__(tier_type=tier_type)
|
|
self._tier = tier
|
|
|
|
def batch_lookup(
|
|
self, keys: list[OffloadKey], req_context: ReqContext
|
|
) -> Iterable[bool]:
|
|
descriptors = [
|
|
(
|
|
_PROBE_ADDR,
|
|
_PROBE_LEN,
|
|
_PROBE_DEV_ID,
|
|
self._tier._file_mapper.get_file_name(k),
|
|
)
|
|
for k in keys
|
|
]
|
|
results = self._tier._agent.query_memory(descriptors, "OBJ", "OBJ")
|
|
return (r is not None for r in results)
|
|
|
|
|
|
class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
|
"""Secondary tier that offloads KV cache blocks to an S3-compatible store.
|
|
|
|
Handles CPU DRAM <-> S3 transfers only. GPU <-> CPU is managed by the
|
|
primary tier. Object keys are formed as ``{prefix}/{hash_shard}/{hash}.bin``.
|
|
"""
|
|
|
|
medium: ClassVar[Medium] = Medium.STORAGE
|
|
|
|
def __init__(
|
|
self,
|
|
offloading_spec: "OffloadingSpec",
|
|
primary_kv_view: memoryview,
|
|
tier_type: str,
|
|
store_config: dict,
|
|
prefix: str = "",
|
|
io_threads: int = 4,
|
|
enable_kv_events: bool = False,
|
|
locality: str | None = None,
|
|
backpressure_detector: BackpressureDetector | None = None,
|
|
):
|
|
"""Args:
|
|
offloading_spec: Offloading configuration.
|
|
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
|
tier_type: Tier type identifier, set by SecondaryTierFactory.
|
|
store_config: Object store connection parameters (see ObjStoreConfig).
|
|
prefix: Key prefix prepended to all object keys.
|
|
io_threads: Number of NIXL I/O threads.
|
|
enable_kv_events: Emit BlockStored KV events for blocks
|
|
successfully stored to this tier. Effective only when KV
|
|
cache events are enabled globally (kv_events_config).
|
|
locality: Whether this tier's storage is LOCAL or REMOTE relative
|
|
to the publishing vLLM instance.
|
|
backpressure_detector: Optional backpressure detector.
|
|
|
|
"""
|
|
super().__init__(
|
|
offloading_spec,
|
|
primary_kv_view,
|
|
tier_type,
|
|
backpressure_detector,
|
|
)
|
|
self.locality = Locality(locality) if locality is not None else None
|
|
|
|
self.events: list[OffloadingEvent] | None = None
|
|
if enable_kv_events:
|
|
if offloading_spec.kv_events_config.enable_kv_cache_events:
|
|
self.events = []
|
|
else:
|
|
logger.warning(
|
|
"enable_kv_events is set on secondary tier '%s' but KV "
|
|
"cache events are disabled globally; the tier will not "
|
|
"emit events.",
|
|
tier_type,
|
|
)
|
|
# Keys of in-flight store jobs, tracked only when events are enabled.
|
|
self._store_job_keys: dict[JobId, list[OffloadKey]] = {}
|
|
# Keys of in-flight load (promotion) jobs, so a failed download can
|
|
# mark its own cached lookup verdicts False (see get_finished_jobs).
|
|
self._load_job_keys: dict[JobId, list[OffloadKey]] = {}
|
|
|
|
agent_config = nixl_agent_config(backends=[])
|
|
self._agent = nixl_agent("ObjAgent", agent_config)
|
|
obj_config = ObjStoreConfig(**store_config)
|
|
params = {**obj_config.to_nixl_params(), "num_threads": str(io_threads)}
|
|
self._agent.create_backend("OBJ", params)
|
|
self._transfers: dict[int, TransferEntry] = {}
|
|
# Buffered results awaiting the next get_finished_jobs() call:
|
|
# submission-time failures + poll-time completions accumulated
|
|
# during drain_jobs().
|
|
self._pending_results: list[JobResult] = []
|
|
self._primary_reg = None
|
|
self._block_size_bytes: int = 0
|
|
root_dir = f"{prefix}/" if prefix else ""
|
|
# Opt in; FileMapper enables it only for a parallelism-invariant block.
|
|
self._file_mapper = FileMapper.from_offloading_spec(
|
|
root_dir, offloading_spec, parallel_agnostic=True
|
|
)
|
|
self._next_obj_dev_id: int = 1 # dev_id=0 is reserved for _exists() probes
|
|
|
|
self._probe_connectivity()
|
|
|
|
base_addr = ctypes.addressof(ctypes.c_char.from_buffer(primary_kv_view))
|
|
assert primary_kv_view.strides is not None
|
|
stride = primary_kv_view.strides[0]
|
|
self._primary_reg = self._agent.register_memory(
|
|
[(base_addr, primary_kv_view.nbytes, NIXL_DEV_ID, "")], "DRAM"
|
|
)
|
|
self._block_size_bytes = stride
|
|
all_blocks = [
|
|
(base_addr + i * stride, stride, NIXL_DEV_ID)
|
|
for i in range(len(primary_kv_view))
|
|
]
|
|
# NIXL_INIT_AGENT marks this as the local side; make_prepped_xfer requires
|
|
# local_xfer_side tagged with NIXL_INIT_AGENT and remote_xfer_side tagged
|
|
# with the peer agent name ("ObjAgent").
|
|
self._dram_prepped_handle: nixl_prepped_dlist_handle = (
|
|
self._agent.prep_xfer_dlist("NIXL_INIT_AGENT", all_blocks, "DRAM")
|
|
)
|
|
|
|
self._lookup_manager = ObjAsyncLookupManager(
|
|
tier=self, tier_type=self.tier_type
|
|
)
|
|
|
|
def _probe_connectivity(self) -> None:
|
|
"""Verify object store connectivity at startup via a NIXL lookup probe.
|
|
|
|
Performs a single exists() check against a synthetic key that will
|
|
never exist. A True/False result confirms the bucket is reachable;
|
|
an exception indicates misconfigured obj store params and raises RuntimeError.
|
|
"""
|
|
probe_key = "__nixl_probe__/connectivity_test"
|
|
try:
|
|
self._exists(probe_key)
|
|
logger.info("Object store tier connectivity probe succeeded")
|
|
except Exception as e:
|
|
raise RuntimeError(
|
|
f"Object store tier connectivity probe failed — check bucket, "
|
|
f"endpoint_override, and scheme. If using explicit credentials "
|
|
f"verify access_key and secret_key; otherwise ensure the AWS "
|
|
f"SDK default credential chain is configured (IAM role, env "
|
|
f"vars, credential file). Error: {e}"
|
|
) from e
|
|
|
|
def _exists(self, obj_key: str) -> bool:
|
|
results = self._agent.query_memory(
|
|
[(_PROBE_ADDR, _PROBE_LEN, _PROBE_DEV_ID, obj_key)], "OBJ", "OBJ"
|
|
)
|
|
return results[0] is not None
|
|
|
|
def _submit_transfer(
|
|
self,
|
|
job_id: int,
|
|
block_ids: np.ndarray,
|
|
obj_keys: Iterable[str],
|
|
op: str,
|
|
) -> None:
|
|
"""Submit an async transfer. op is 'WRITE' (store) or 'READ' (load)."""
|
|
# NIXL takes indices as int32; a wider dtype silently costs a conversion.
|
|
assert block_ids.dtype == np.int32
|
|
# The OBJ backend maps devId -> obj_key. All descriptors must have
|
|
# unique devIds or later registrations overwrite earlier ones.
|
|
nixl_files = [
|
|
(0, self._block_size_bytes, dev_id, key)
|
|
for dev_id, key in enumerate(obj_keys, self._next_obj_dev_id)
|
|
]
|
|
self._next_obj_dev_id += len(nixl_files)
|
|
|
|
files_desc = self._agent.register_memory(nixl_files, "OBJ")
|
|
if files_desc is None:
|
|
logger.warning("register_memory (OBJ) failed for job %d", job_id)
|
|
self._pending_results.append(JobResult(job_id=job_id, success=False))
|
|
return
|
|
|
|
obj_handle = self._agent.prep_xfer_dlist("ObjAgent", files_desc.trim())
|
|
if not obj_handle:
|
|
logger.warning("prep_xfer_dlist (OBJ) failed for job %d", job_id)
|
|
self._agent.deregister_memory(files_desc)
|
|
self._pending_results.append(JobResult(job_id=job_id, success=False))
|
|
return
|
|
|
|
xfer_handle = self._agent.make_prepped_xfer(
|
|
op,
|
|
self._dram_prepped_handle,
|
|
block_ids,
|
|
obj_handle,
|
|
np.arange(len(nixl_files), dtype=np.int32),
|
|
)
|
|
if not xfer_handle:
|
|
logger.warning("make_prepped_xfer failed for job %d", job_id)
|
|
self._agent.release_dlist_handle(obj_handle)
|
|
self._agent.deregister_memory(files_desc)
|
|
self._pending_results.append(JobResult(job_id=job_id, success=False))
|
|
return
|
|
|
|
state = self._agent.transfer(xfer_handle)
|
|
if state == "ERR":
|
|
logger.warning("agent.transfer failed for job %d", job_id)
|
|
self._agent.release_dlist_handle(obj_handle)
|
|
self._agent.deregister_memory(files_desc)
|
|
self._agent.release_xfer_handle(xfer_handle)
|
|
self._pending_results.append(JobResult(job_id=job_id, success=False))
|
|
return
|
|
|
|
self._transfers[job_id] = TransferEntry(
|
|
xfer_handle=xfer_handle,
|
|
files_desc=files_desc,
|
|
obj_handle=obj_handle,
|
|
)
|
|
|
|
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
|
result = self._lookup_manager.lookup(key, req_context)
|
|
if result is None:
|
|
return LookupResult.RETRY
|
|
return LookupResult.HIT if result else LookupResult.MISS
|
|
|
|
def submit_store(self, job_metadata: TransferJob) -> None:
|
|
if self.events is not None:
|
|
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
|
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
|
self._submit_transfer(
|
|
job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_WRITE
|
|
)
|
|
|
|
def submit_load(self, job_metadata: TransferJob) -> None:
|
|
self._load_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
|
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
|
self._submit_transfer(
|
|
job_metadata.job_id, job_metadata.chunk_ids, obj_keys, NIXL_READ
|
|
)
|
|
|
|
def on_request_finished(self, req_context: ReqContext) -> None:
|
|
self._lookup_manager.cleanup(req_context.req_id)
|
|
|
|
def on_schedule_end(self, context: ScheduleEndContext) -> None:
|
|
self._lookup_manager.flush()
|
|
|
|
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
|
return RequestOffloadingContext()
|
|
|
|
def _poll_active_transfers(self) -> None:
|
|
"""Poll all in-flight transfers once; move newly-completed (success or
|
|
failure) into ``_pending_results`` and release their NIXL handles."""
|
|
for job_id, entry in list(self._transfers.items()):
|
|
try:
|
|
state = self._agent.check_xfer_state(entry.xfer_handle)
|
|
except Exception as exc:
|
|
success = False
|
|
logger.warning("check_xfer_state raised for job %d: %s", job_id, exc)
|
|
else:
|
|
if state == NIXL_PROC:
|
|
continue
|
|
if state == NIXL_DONE:
|
|
success = True
|
|
else:
|
|
success = False
|
|
logger.warning("transfer failed job=%d state=%s", job_id, state)
|
|
|
|
transfer_time = None
|
|
if success:
|
|
telemetry = self._agent.get_xfer_telemetry(entry.xfer_handle)
|
|
transfer_time = telemetry.xferDuration / 1e6
|
|
|
|
try:
|
|
self._agent.release_xfer_handle(entry.xfer_handle)
|
|
except Exception as exc:
|
|
# Keep the entry until NIXL confirms that the transfer handle
|
|
# can be released. The transfer may still access primary-tier
|
|
# memory, so publishing its result would allow unsafe reuse.
|
|
logger.warning("release_xfer_handle failed for job %d: %s", job_id, exc)
|
|
continue
|
|
|
|
# Once the transfer handle is released, these remaining cleanup
|
|
# failures must not suppress the job completion. They can leak
|
|
# NIXL metadata, but cannot leave an active data transfer behind.
|
|
try:
|
|
self._agent.release_dlist_handle(entry.obj_handle)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"release_dlist_handle failed for job %d: %s", job_id, exc
|
|
)
|
|
try:
|
|
self._agent.deregister_memory(entry.files_desc)
|
|
except Exception as exc:
|
|
logger.warning("deregister_memory failed for job %d: %s", job_id, exc)
|
|
|
|
del self._transfers[job_id]
|
|
self._pending_results.append(
|
|
JobResult(
|
|
job_id=job_id,
|
|
success=success,
|
|
transfer_time=transfer_time,
|
|
)
|
|
)
|
|
|
|
def get_finished_jobs(self) -> Iterable[JobResult]:
|
|
"""Poll transfers; a failed promotion marks its cached verdicts False
|
|
here (scheduler thread)."""
|
|
self._poll_active_transfers()
|
|
results = self._pending_results
|
|
self._pending_results = []
|
|
for result in results:
|
|
if self.events is not None:
|
|
keys = self._store_job_keys.pop(result.job_id, None)
|
|
if result.success and keys:
|
|
self.events.append(
|
|
OffloadingEvent(
|
|
keys=keys,
|
|
medium=self.medium,
|
|
removed=False,
|
|
locality=self.locality,
|
|
)
|
|
)
|
|
# Mark only the keys that did not load as a miss; the request
|
|
# recomputes them. The miss is per-request (cleared when the request
|
|
# finishes), so other requests still HIT the blocks that loaded
|
|
# fine. nixl reports the batch as a whole (successful_keys is None),
|
|
# so today this marks all keys; the subtraction keeps it correct if
|
|
# partial results are ever reported.
|
|
load_keys = self._load_job_keys.pop(result.job_id, None)
|
|
if load_keys is not None and not result.success:
|
|
successful = set(result.successful_keys or ())
|
|
failed = [k for k in load_keys if k not in successful]
|
|
self._lookup_manager.mark_miss(failed)
|
|
return results
|
|
|
|
def take_events(self) -> Iterable[OffloadingEvent]:
|
|
if self.events is not None:
|
|
yield from self.events
|
|
self.events.clear()
|
|
|
|
def drain_jobs(self) -> None:
|
|
"""Block until every submitted transfer has completed or failed.
|
|
|
|
nixl exposes only ``check_xfer_state`` (poll-based), so this loops
|
|
until ``_transfers`` is empty. Results accumulate in
|
|
``_pending_results`` and are surfaced by the next
|
|
``get_finished_jobs()`` call.
|
|
"""
|
|
start = time.monotonic()
|
|
warned = False
|
|
while self._transfers:
|
|
self._poll_active_transfers()
|
|
if not self._transfers:
|
|
break
|
|
if not warned and time.monotonic() - start > 5.0:
|
|
logger.warning(
|
|
"ObjectStoreSecondaryTierManager.drain_jobs: still "
|
|
"draining after 5s (%d transfers in flight); a stuck "
|
|
"transfer will block the engine.",
|
|
len(self._transfers),
|
|
)
|
|
warned = True
|
|
time.sleep(0.001)
|
|
|
|
def shutdown(self) -> None:
|
|
self._lookup_manager.shutdown()
|
|
for job_id, entry in self._transfers.items():
|
|
try:
|
|
self._agent.release_xfer_handle(entry.xfer_handle)
|
|
except Exception as exc:
|
|
logger.warning("release_xfer_handle failed for job %d: %s", job_id, exc)
|
|
try:
|
|
self._agent.release_dlist_handle(entry.obj_handle)
|
|
except Exception as exc:
|
|
logger.warning(
|
|
"release_dlist_handle failed for job %d: %s", job_id, exc
|
|
)
|
|
try:
|
|
self._agent.deregister_memory(entry.files_desc)
|
|
except Exception as exc:
|
|
logger.warning("deregister_memory failed for job %d: %s", job_id, exc)
|
|
self._transfers.clear()
|
|
if self._dram_prepped_handle is not None:
|
|
try:
|
|
self._agent.release_dlist_handle(self._dram_prepped_handle)
|
|
except Exception as exc:
|
|
logger.warning("failed to release DRAM prepped handle: %s", exc)
|
|
self._dram_prepped_handle = None
|
|
if self._primary_reg is not None:
|
|
try:
|
|
self._agent.deregister_memory(self._primary_reg)
|
|
except Exception as exc:
|
|
logger.warning("failed to deregister primary buffer: %s", exc)
|
|
self._primary_reg = None
|