[kv_offload] Emit tier-owned BlockStored events from FS/OBJ secondary tiers (#47923)
Signed-off-by: Change72 <changg@nvidia.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude Fable 5
Or Ozeri
parent
88e5e2c57b
commit
2d814a0082
@@ -44,6 +44,8 @@ class KVCacheEvent(
|
||||
|
||||
MEDIUM_GPU = "GPU"
|
||||
MEDIUM_CPU = "CPU"
|
||||
MEDIUM_FS = "FS"
|
||||
MEDIUM_OBJ = "OBJ"
|
||||
|
||||
|
||||
class BlockStored(KVCacheEvent):
|
||||
|
||||
@@ -19,7 +19,7 @@ import functools
|
||||
import json
|
||||
import os
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
try:
|
||||
from vllm.fs_io_C import batch_lookup as batch_lookup_C
|
||||
@@ -30,11 +30,18 @@ except ImportError:
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_FS
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
RequestOffloadingContext,
|
||||
@@ -92,6 +99,8 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
content.
|
||||
"""
|
||||
|
||||
medium: ClassVar[str] = MEDIUM_FS
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: "OffloadingSpec",
|
||||
@@ -100,6 +109,7 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
root_dir: str,
|
||||
n_read_threads: int = 16,
|
||||
n_write_threads: int = 16,
|
||||
enable_kv_events: bool = False,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
@@ -110,9 +120,26 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
root_dir: Root directory for block files.
|
||||
n_read_threads: Number of read-priority I/O threads.
|
||||
n_write_threads: Number of write-priority I/O threads.
|
||||
enable_kv_events: Emit BlockStored KV events for blocks
|
||||
successfully stored to this tier. Effective only when KV
|
||||
cache events are enabled globally (kv_events_config).
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
|
||||
self.events: list[OffloadingEvent] | None = None
|
||||
if enable_kv_events:
|
||||
if offloading_spec.kv_events_config.enable_kv_cache_events:
|
||||
self.events = []
|
||||
else:
|
||||
logger.warning(
|
||||
"enable_kv_events is set on secondary tier '%s' but KV "
|
||||
"cache events are disabled globally; the tier will not "
|
||||
"emit events.",
|
||||
tier_type,
|
||||
)
|
||||
# Keys of in-flight store jobs, tracked only when events are enabled.
|
||||
self._store_job_keys: dict[JobId, list[OffloadKey]] = {}
|
||||
|
||||
# Extract block size from primary view
|
||||
assert primary_kv_view.strides is not None, (
|
||||
"primary_kv_view.strides cannot be None"
|
||||
@@ -157,6 +184,8 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
if self.events is not None:
|
||||
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||
tasks = (
|
||||
functools.partial(
|
||||
store_block,
|
||||
@@ -188,10 +217,22 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
"""
|
||||
Collect completed jobs from the finished-jobs queue.
|
||||
"""
|
||||
return (
|
||||
JobResult(job_id=job_id, success=success)
|
||||
for job_id, success in self._pool.get_finished()
|
||||
)
|
||||
results = []
|
||||
for job_id, success in self._pool.get_finished():
|
||||
if self.events is not None:
|
||||
keys = self._store_job_keys.pop(job_id, None)
|
||||
if success and keys:
|
||||
self.events.append(
|
||||
OffloadingEvent(keys=keys, medium=self.medium, removed=False)
|
||||
)
|
||||
results.append(JobResult(job_id=job_id, success=success))
|
||||
return results
|
||||
|
||||
@override
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
if self.events is not None:
|
||||
yield from self.events
|
||||
self.events.clear()
|
||||
|
||||
@override
|
||||
def drain_jobs(self) -> None:
|
||||
|
||||
@@ -5,15 +5,22 @@
|
||||
import ctypes
|
||||
import time
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING, NamedTuple
|
||||
from typing import TYPE_CHECKING, ClassVar, NamedTuple
|
||||
|
||||
from vllm.distributed.kv_events import MEDIUM_OBJ
|
||||
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
|
||||
from vllm.distributed.nixl_utils import nixl_agent_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobId,
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
RequestOffloadingContext,
|
||||
@@ -90,6 +97,8 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
primary tier. Object keys are formed as ``{prefix}/{hash_shard}/{hash}.bin``.
|
||||
"""
|
||||
|
||||
medium: ClassVar[str] = MEDIUM_OBJ
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
offloading_spec: "OffloadingSpec",
|
||||
@@ -98,8 +107,36 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
store_config: dict,
|
||||
prefix: str = "",
|
||||
io_threads: int = 4,
|
||||
enable_kv_events: bool = False,
|
||||
):
|
||||
"""
|
||||
Args:
|
||||
offloading_spec: Offloading configuration.
|
||||
primary_kv_view: Memoryview of the primary tier's CPU KV cache.
|
||||
tier_type: Tier type identifier, set by SecondaryTierFactory.
|
||||
store_config: Object store connection parameters (see ObjStoreConfig).
|
||||
prefix: Key prefix prepended to all object keys.
|
||||
io_threads: Number of NIXL I/O threads.
|
||||
enable_kv_events: Emit BlockStored KV events for blocks
|
||||
successfully stored to this tier. Effective only when KV
|
||||
cache events are enabled globally (kv_events_config).
|
||||
"""
|
||||
super().__init__(offloading_spec, primary_kv_view, tier_type)
|
||||
|
||||
self.events: list[OffloadingEvent] | None = None
|
||||
if enable_kv_events:
|
||||
if offloading_spec.kv_events_config.enable_kv_cache_events:
|
||||
self.events = []
|
||||
else:
|
||||
logger.warning(
|
||||
"enable_kv_events is set on secondary tier '%s' but KV "
|
||||
"cache events are disabled globally; the tier will not "
|
||||
"emit events.",
|
||||
tier_type,
|
||||
)
|
||||
# Keys of in-flight store jobs, tracked only when events are enabled.
|
||||
self._store_job_keys: dict[JobId, list[OffloadKey]] = {}
|
||||
|
||||
agent_config = nixl_agent_config(backends=[])
|
||||
self._agent = nixl_agent("ObjAgent", agent_config)
|
||||
obj_config = ObjStoreConfig(**store_config)
|
||||
@@ -231,6 +268,8 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
return LookupResult.HIT if result else LookupResult.MISS
|
||||
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
if self.events is not None:
|
||||
self._store_job_keys[job_metadata.job_id] = list(job_metadata.keys)
|
||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||
self._submit_transfer(
|
||||
job_metadata.job_id, job_metadata.block_ids, obj_keys, NIXL_WRITE
|
||||
@@ -279,8 +318,20 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
self._poll_active_transfers()
|
||||
results = self._pending_results
|
||||
self._pending_results = []
|
||||
if self.events is not None:
|
||||
for result in results:
|
||||
keys = self._store_job_keys.pop(result.job_id, None)
|
||||
if result.success and keys:
|
||||
self.events.append(
|
||||
OffloadingEvent(keys=keys, medium=self.medium, removed=False)
|
||||
)
|
||||
return results
|
||||
|
||||
def take_events(self) -> Iterable[OffloadingEvent]:
|
||||
if self.events is not None:
|
||||
yield from self.events
|
||||
self.events.clear()
|
||||
|
||||
def drain_jobs(self) -> None:
|
||||
"""Block until every submitted transfer has completed or failed.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user