[KV Offloading] Replace bool|None lookup return with LookupResult enum (#46363)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Ronen Schaffer
2026-06-24 18:06:08 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 7f99e80c3b
commit bb61177e49
14 changed files with 323 additions and 172 deletions
@@ -35,6 +35,7 @@ from vllm.v1.kv_cache_interface import (
)
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
LookupResult,
OffloadingManager,
OffloadingSpec,
OffloadKey,
@@ -393,15 +394,18 @@ class OffloadingConnectorScheduler:
hit_count = 0
defer_lookup = False
for key in keys:
result = self.manager.lookup(key, req_context)
if result is None:
defer_lookup = True
# continue lookup to allow manager to kick-off async lookups
# for all blocks (until a miss is detected)
result = True
if not result:
break
hit_count += 1
match self.manager.lookup(key, req_context):
case LookupResult.HIT:
hit_count += 1
case LookupResult.HIT_PENDING:
defer_lookup = True
hit_count += 1
case LookupResult.RETRY:
# Don't break: keep scanning to let manager kick off
# async lookups (until a miss is detected).
defer_lookup = True
case LookupResult.MISS:
break
return hit_count if not defer_lookup else None
def _sliding_window_lookup(
@@ -416,18 +420,25 @@ class OffloadingConnectorScheduler:
defer_lookup = False
consecutive_hits = 0
for idx in range(len(keys) - 1, -1, -1):
result = self.manager.lookup(keys[idx], req_context)
if result is None:
defer_lookup = True
# continue lookup to allow manager to kick-off async lookups
# for all blocks (until a hit is detected)
result = False
if not result:
consecutive_hits = 0
else:
consecutive_hits += 1
if consecutive_hits == sliding_window_size:
return idx + sliding_window_size if not defer_lookup else None
match self.manager.lookup(keys[idx], req_context):
case LookupResult.HIT:
consecutive_hits += 1
case LookupResult.HIT_PENDING:
# Block is in cache, just not readable yet — counts
# as hit for the consecutive streak. Don't break:
# keep scanning to let manager kick off async lookups.
defer_lookup = True
consecutive_hits += 1
case LookupResult.RETRY:
# Block location uncertain — does not count as hit.
# Don't break: keep scanning to let manager kick off
# async lookups.
defer_lookup = True
consecutive_hits = 0
case LookupResult.MISS:
consecutive_hits = 0
if consecutive_hits == sliding_window_size:
return idx + sliding_window_size if not defer_lookup else None
return consecutive_hits if not defer_lookup else None
def _touch(self, req_status: RequestOffloadState):
+14 -6
View File
@@ -7,7 +7,7 @@ Core abstractions for KV cache offloading in vLLM v1.
from abc import ABC, abstractmethod
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass
from enum import Enum
from enum import Enum, auto
from typing import TYPE_CHECKING, Any, NewType
import numpy as np
@@ -53,6 +53,15 @@ class ReqContext:
kv_transfer_params: dict[str, Any] | None = None
class LookupResult(Enum):
"""Result of OffloadingManager.lookup()."""
MISS = auto()
HIT = auto()
HIT_PENDING = auto()
RETRY = auto()
class OffloadPolicy(Enum):
# Offload only newly-computed blocks as they arrive; prefix-hit
# blocks (already offloaded by a prior request) are skipped.
@@ -158,7 +167,7 @@ class OffloadingKVEventsConfig:
class OffloadingManager(ABC):
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
"""
Checks whether a single block is offloaded and ready to be read.
@@ -167,10 +176,9 @@ class OffloadingManager(ABC):
req_context: per-request context (e.g. kv_transfer_params).
Returns:
True if the block is offloaded and ready, False if not,
or None if the lookup should be retried later.
Returning None will delay the request handling by the vLLM
scheduler.
HIT if the block is offloaded and ready, MISS if not found,
HIT_PENDING if found but not yet readable, or RETRY if the
lookup should be retried later.
"""
pass
+5 -4
View File
@@ -11,6 +11,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
)
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
OffloadingEvent,
OffloadingManager,
OffloadKey,
@@ -112,7 +113,7 @@ class CPUOffloadingManager(OffloadingManager):
return RequestOffloadingContext()
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
if self.counts is not None:
if key in self.counts:
self.counts.move_to_end(key)
@@ -123,10 +124,10 @@ class CPUOffloadingManager(OffloadingManager):
self.counts[key] = 1
block = self._policy.get(key)
if block is None:
return False
return LookupResult.MISS
if not block.is_ready:
return None # write in-flight; caller should retry
return True
return LookupResult.HIT_PENDING
return LookupResult.HIT
@override
def prepare_load(
+5 -4
View File
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
import numpy as np
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadingMetricMetadata,
OffloadKey,
ReqContext,
@@ -79,7 +80,7 @@ class SecondaryTierManager(ABC):
self.tier_type = tier_type
@abstractmethod
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
"""
Check whether a block exists in this secondary tier.
@@ -88,9 +89,9 @@ class SecondaryTierManager(ABC):
req_context: per-request context (e.g. kv_transfer_params).
Returns:
True if the block is present and ready,
False if not found,
or None if the block is being transferred (retry later).
HIT if the block is present and ready,
MISS if not found,
or RETRY if the block is being transferred (retry later).
"""
pass
@@ -15,7 +15,12 @@ from typing import TYPE_CHECKING
from typing_extensions import override
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, RequestOffloadingContext
from vllm.v1.kv_offload.base import (
LookupResult,
OffloadKey,
ReqContext,
RequestOffloadingContext,
)
from vllm.v1.kv_offload.tiering.base import (
JobMetadata,
JobResult,
@@ -67,7 +72,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
self.completed_jobs: list[JobResult] = []
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
"""
Check whether a block exists in this secondary tier.
@@ -76,9 +81,9 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
req_context: Per-request context.
Returns:
True if the block is present, False if not found.
HIT if the block is present, MISS if not found.
"""
return key in self.blocks
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
@override
def submit_store(self, job_metadata: JobMetadata) -> None:
+6 -3
View File
@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING
from typing_extensions import override
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.base import (
@@ -137,8 +137,11 @@ class FileSystemTierManager(SecondaryTierManager):
return RequestOffloadingContext()
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
return self._lookup_manager.lookup(key, req_context)
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
result = self._lookup_manager.lookup(key, req_context)
if result is None:
return LookupResult.RETRY
return LookupResult.HIT if result else LookupResult.MISS
@override
def submit_store(self, job_metadata: JobMetadata) -> None:
+24 -20
View File
@@ -32,6 +32,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import (
LoadStoreSpec,
LookupResult,
OffloadingEvent,
OffloadingManager,
OffloadKey,
@@ -233,7 +234,7 @@ class TieringOffloadingManager(OffloadingManager):
)
@override
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
"""
Check whether a single block is offloaded and ready.
@@ -248,33 +249,34 @@ class TieringOffloadingManager(OffloadingManager):
req_context: Per-request context.
Returns:
True — block is ready in the primary tier.
None — block found but not yet ready (primary in-flight,
promotion started, or a secondary tier is busy).
False — block not found in any tier, or primary is full
and cannot accept a promotion.
HIT — block is ready in the primary tier.
HIT_PENDING — block found but not yet readable (write
in-flight on the primary tier).
RETRY — promotion started or a secondary tier is busy.
MISS — block not found in any tier, or primary is full
and cannot accept a promotion.
"""
self._maybe_process_finished_jobs()
primary_hit = self.primary_tier.lookup(key, req_context)
if primary_hit is True:
return True
if primary_hit is None:
return None
if primary_hit is LookupResult.HIT:
return LookupResult.HIT
if primary_hit is LookupResult.HIT_PENDING:
return LookupResult.HIT_PENDING
any_none = False
any_retry = False
for tier in self.secondary_tiers:
result = tier.lookup(key, req_context)
if result is True:
if result is LookupResult.HIT:
if not self._initiate_promotion(tier, key, req_context):
return False # primary full, block unavailable
return None # promotion started, retry later
if result is None:
any_none = True
return LookupResult.MISS
return LookupResult.RETRY
if result is LookupResult.RETRY:
any_retry = True
if any_none:
return None
return False
if any_retry:
return LookupResult.RETRY
return LookupResult.MISS
def _initiate_promotion(
self,
@@ -467,7 +469,9 @@ class TieringOffloadingManager(OffloadingManager):
"""
# Filter out keys that are not ready in primary (e.g. in-flight)
ready_keys = tuple(
k for k in keys if self.primary_tier.lookup(k, req_context) is True
k
for k in keys
if self.primary_tier.lookup(k, req_context) is LookupResult.HIT
)
if not ready_keys:
return
+6 -3
View File
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, NamedTuple
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
from vllm.distributed.nixl_utils import nixl_agent_config
from vllm.logger import init_logger
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
from vllm.v1.kv_offload.file_mapper import FileMapper
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
from vllm.v1.kv_offload.tiering.base import (
@@ -221,8 +221,11 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
self._transfers[job_id] = TransferEntry(xfer_handle, files_desc, obj_handle)
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
return self._lookup_manager.lookup(key, req_context)
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
result = self._lookup_manager.lookup(key, req_context)
if result is None:
return LookupResult.RETRY
return LookupResult.HIT if result else LookupResult.MISS
def submit_store(self, job_metadata: JobMetadata) -> None:
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)