[kv_offload] Emit tier-owned BlockStored events from FS/OBJ secondary tiers (#47923)

Signed-off-by: Change72 <changg@nvidia.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Chang Guo
2026-07-10 06:17:23 +03:00
committed by GitHub
co-authored by Claude Fable 5 Or Ozeri
parent 88e5e2c57b
commit 2d814a0082
6 changed files with 433 additions and 9 deletions
+2
View File
@@ -44,6 +44,8 @@ class KVCacheEvent(
MEDIUM_GPU = "GPU"
MEDIUM_CPU = "CPU"
MEDIUM_FS = "FS"
MEDIUM_OBJ = "OBJ"
class BlockStored(KVCacheEvent):
+47 -6
View File
@@ -19,7 +19,7 @@ import functools
import json
import os
from collections.abc import Iterable
from typing import TYPE_CHECKING
from typing import TYPE_CHECKING, ClassVar
try:
from vllm.fs_io_C import batch_lookup as batch_lookup_C
@@ -30,11 +30,18 @@ except ImportError:
from typing_extensions import override
from vllm.distributed.kv_events import MEDIUM_FS
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingEvent,
OffloadKey,
ReqContext,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobMetadata,
JobResult,
RequestOffloadingContext,
@@ -92,6 +99,8 @@ class FileSystemTierManager(SecondaryTierManager):
content.
"""
medium: ClassVar[str] = MEDIUM_FS
def __init__(
self,
offloading_spec: "OffloadingSpec",
@@ -100,6 +109,7 @@ class FileSystemTierManager(SecondaryTierManager):
root_dir: str,
n_read_threads: int = 16,
n_write_threads: int = 16,
enable_kv_events: bool = False,
):
"""
Args:
@@ -110,9 +120,26 @@ class FileSystemTierManager(SecondaryTierManager):
root_dir: Root directory for block files.
n_read_threads: Number of read-priority I/O threads.
n_write_threads: Number of write-priority I/O threads.
enable_kv_events: Emit BlockStored KV events for blocks
successfully stored to this tier. Effective only when KV
cache events are enabled globally (kv_events_config).
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
self.events: list[OffloadingEvent] | None = None
if enable_kv_events:
if offloading_spec.kv_events_config.enable_kv_cache_events:
self.events = []
else:
logger.warning(
"enable_kv_events is set on secondary tier '%s' but KV "
"cache events are disabled globally; the tier will not "
"emit events.",
tier_type,
)
# Keys of in-flight store jobs, tracked only when events are enabled.
self._store_job_keys: dict[JobId, list[OffloadKey]] = {}
# Extract block size from primary view
assert primary_kv_view.strides is not None, (
"primary_kv_view.strides cannot be None"
@@ -157,6 +184,8 @@ class FileSystemTierManager(SecondaryTierManager):
@override
def submit_store(self, job_metadata: JobMetadata) -> None:
if self.events is not None:
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
tasks = (
functools.partial(
store_block,
@@ -188,10 +217,22 @@ class FileSystemTierManager(SecondaryTierManager):
"""
Collect completed jobs from the finished-jobs queue.
"""
return (
JobResult(job_id=job_id, success=success)
for job_id, success in self._pool.get_finished()
)
results = []
for job_id, success in self._pool.get_finished():
if self.events is not None:
keys = self._store_job_keys.pop(job_id, None)
if success and keys:
self.events.append(
OffloadingEvent(keys=keys, medium=self.medium, removed=False)
)
results.append(JobResult(job_id=job_id, success=success))
return results
@override
def take_events(self) -> Iterable[OffloadingEvent]:
if self.events is not None:
yield from self.events
self.events.clear()
@override
def drain_jobs(self) -> None:
+53 -2
View File
@@ -5,15 +5,22 @@
import ctypes
import time
from collections.abc import Iterable
from typing import TYPE_CHECKING, NamedTuple
from typing import TYPE_CHECKING, ClassVar, NamedTuple
from vllm.distributed.kv_events import MEDIUM_OBJ
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
from vllm.distributed.nixl_utils import nixl_agent_config
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingEvent,
OffloadKey,
ReqContext,
)
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.base import (
JobId,
JobMetadata,
JobResult,
RequestOffloadingContext,
@@ -90,6 +97,8 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
primary tier. Object keys are formed as ``{prefix}/{hash_shard}/{hash}.bin``.
"""
medium: ClassVar[str] = MEDIUM_OBJ
def __init__(
self,
offloading_spec: "OffloadingSpec",
@@ -98,8 +107,36 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
store_config: dict,
prefix: str = "",
io_threads: int = 4,
enable_kv_events: bool = False,
):
"""
Args:
offloading_spec: Offloading configuration.
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
tier_type: Tier type identifier, set by SecondaryTierFactory.
store_config: Object store connection parameters (see ObjStoreConfig).
prefix: Key prefix prepended to all object keys.
io_threads: Number of NIXL I/O threads.
enable_kv_events: Emit BlockStored KV events for blocks
successfully stored to this tier. Effective only when KV
cache events are enabled globally (kv_events_config).
"""
super().__init__(offloading_spec, primary_kv_view, tier_type)
self.events: list[OffloadingEvent] | None = None
if enable_kv_events:
if offloading_spec.kv_events_config.enable_kv_cache_events:
self.events = []
else:
logger.warning(
"enable_kv_events is set on secondary tier '%s' but KV "
"cache events are disabled globally; the tier will not "
"emit events.",
tier_type,
)
# Keys of in-flight store jobs, tracked only when events are enabled.
self._store_job_keys: dict[JobId, list[OffloadKey]] = {}
agent_config = nixl_agent_config(backends=[])
self._agent = nixl_agent("ObjAgent", agent_config)
obj_config = ObjStoreConfig(**store_config)
@@ -231,6 +268,8 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
return LookupResult.HIT if result else LookupResult.MISS
def submit_store(self, job_metadata: JobMetadata) -> None:
if self.events is not None:
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
self._submit_transfer(
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE
@@ -279,8 +318,20 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
self._poll_active_transfers()
results = self._pending_results
self._pending_results = []
if self.events is not None:
for result in results:
keys = self._store_job_keys.pop(result.job_id, None)
if result.success and keys:
self.events.append(
OffloadingEvent(keys=keys, medium=self.medium, removed=False)
)
return results
def take_events(self) -> Iterable[OffloadingEvent]:
if self.events is not None:
yield from self.events
self.events.clear()
def drain_jobs(self) -> None:
"""Block until every submitted transfer has completed or failed.