[KV Offloading] Replace bool|None lookup return with LookupResult enum (#46363)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Or Ozeri
parent
7f99e80c3b
commit
bb61177e49
@@ -35,6 +35,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
GPULoadStoreSpec,
|
||||
LookupResult,
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadKey,
|
||||
@@ -393,15 +394,18 @@ class OffloadingConnectorScheduler:
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
for key in keys:
|
||||
result = self.manager.lookup(key, req_context)
|
||||
if result is None:
|
||||
defer_lookup = True
|
||||
# continue lookup to allow manager to kick-off async lookups
|
||||
# for all blocks (until a miss is detected)
|
||||
result = True
|
||||
if not result:
|
||||
break
|
||||
hit_count += 1
|
||||
match self.manager.lookup(key, req_context):
|
||||
case LookupResult.HIT:
|
||||
hit_count += 1
|
||||
case LookupResult.HIT_PENDING:
|
||||
defer_lookup = True
|
||||
hit_count += 1
|
||||
case LookupResult.RETRY:
|
||||
# Don't break: keep scanning to let manager kick off
|
||||
# async lookups (until a miss is detected).
|
||||
defer_lookup = True
|
||||
case LookupResult.MISS:
|
||||
break
|
||||
return hit_count if not defer_lookup else None
|
||||
|
||||
def _sliding_window_lookup(
|
||||
@@ -416,18 +420,25 @@ class OffloadingConnectorScheduler:
|
||||
defer_lookup = False
|
||||
consecutive_hits = 0
|
||||
for idx in range(len(keys) - 1, -1, -1):
|
||||
result = self.manager.lookup(keys[idx], req_context)
|
||||
if result is None:
|
||||
defer_lookup = True
|
||||
# continue lookup to allow manager to kick-off async lookups
|
||||
# for all blocks (until a hit is detected)
|
||||
result = False
|
||||
if not result:
|
||||
consecutive_hits = 0
|
||||
else:
|
||||
consecutive_hits += 1
|
||||
if consecutive_hits == sliding_window_size:
|
||||
return idx + sliding_window_size if not defer_lookup else None
|
||||
match self.manager.lookup(keys[idx], req_context):
|
||||
case LookupResult.HIT:
|
||||
consecutive_hits += 1
|
||||
case LookupResult.HIT_PENDING:
|
||||
# Block is in cache, just not readable yet — counts
|
||||
# as hit for the consecutive streak. Don't break:
|
||||
# keep scanning to let manager kick off async lookups.
|
||||
defer_lookup = True
|
||||
consecutive_hits += 1
|
||||
case LookupResult.RETRY:
|
||||
# Block location uncertain — does not count as hit.
|
||||
# Don't break: keep scanning to let manager kick off
|
||||
# async lookups.
|
||||
defer_lookup = True
|
||||
consecutive_hits = 0
|
||||
case LookupResult.MISS:
|
||||
consecutive_hits = 0
|
||||
if consecutive_hits == sliding_window_size:
|
||||
return idx + sliding_window_size if not defer_lookup else None
|
||||
return consecutive_hits if not defer_lookup else None
|
||||
|
||||
def _touch(self, req_status: RequestOffloadState):
|
||||
|
||||
@@ -7,7 +7,7 @@ Core abstractions for KV cache offloading in vLLM v1.
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from enum import Enum, auto
|
||||
from typing import TYPE_CHECKING, Any, NewType
|
||||
|
||||
import numpy as np
|
||||
@@ -53,6 +53,15 @@ class ReqContext:
|
||||
kv_transfer_params: dict[str, Any] | None = None
|
||||
|
||||
|
||||
class LookupResult(Enum):
|
||||
"""Result of OffloadingManager.lookup()."""
|
||||
|
||||
MISS = auto()
|
||||
HIT = auto()
|
||||
HIT_PENDING = auto()
|
||||
RETRY = auto()
|
||||
|
||||
|
||||
class OffloadPolicy(Enum):
|
||||
# Offload only newly-computed blocks as they arrive; prefix-hit
|
||||
# blocks (already offloaded by a prior request) are skipped.
|
||||
@@ -158,7 +167,7 @@ class OffloadingKVEventsConfig:
|
||||
|
||||
class OffloadingManager(ABC):
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""
|
||||
Checks whether a single block is offloaded and ready to be read.
|
||||
|
||||
@@ -167,10 +176,9 @@ class OffloadingManager(ABC):
|
||||
req_context: per-request context (e.g. kv_transfer_params).
|
||||
|
||||
Returns:
|
||||
True if the block is offloaded and ready, False if not,
|
||||
or None if the lookup should be retried later.
|
||||
Returning None will delay the request handling by the vLLM
|
||||
scheduler.
|
||||
HIT if the block is offloaded and ready, MISS if not found,
|
||||
HIT_PENDING if found but not yet readable, or RETRY if the
|
||||
lookup should be retried later.
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
)
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadKey,
|
||||
@@ -112,7 +113,7 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
if self.counts is not None:
|
||||
if key in self.counts:
|
||||
self.counts.move_to_end(key)
|
||||
@@ -123,10 +124,10 @@ class CPUOffloadingManager(OffloadingManager):
|
||||
self.counts[key] = 1
|
||||
block = self._policy.get(key)
|
||||
if block is None:
|
||||
return False
|
||||
return LookupResult.MISS
|
||||
if not block.is_ready:
|
||||
return None # write in-flight; caller should retry
|
||||
return True
|
||||
return LookupResult.HIT_PENDING
|
||||
return LookupResult.HIT
|
||||
|
||||
@override
|
||||
def prepare_load(
|
||||
|
||||
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any
|
||||
import numpy as np
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadingMetricMetadata,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
@@ -79,7 +80,7 @@ class SecondaryTierManager(ABC):
|
||||
self.tier_type = tier_type
|
||||
|
||||
@abstractmethod
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
|
||||
@@ -88,9 +89,9 @@ class SecondaryTierManager(ABC):
|
||||
req_context: per-request context (e.g. kv_transfer_params).
|
||||
|
||||
Returns:
|
||||
True if the block is present and ready,
|
||||
False if not found,
|
||||
or None if the block is being transferred (retry later).
|
||||
HIT if the block is present and ready,
|
||||
MISS if not found,
|
||||
or RETRY if the block is being transferred (retry later).
|
||||
"""
|
||||
pass
|
||||
|
||||
|
||||
@@ -15,7 +15,12 @@ from typing import TYPE_CHECKING
|
||||
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext, RequestOffloadingContext
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LookupResult,
|
||||
OffloadKey,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
JobMetadata,
|
||||
JobResult,
|
||||
@@ -67,7 +72,7 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
self.completed_jobs: list[JobResult] = []
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""
|
||||
Check whether a block exists in this secondary tier.
|
||||
|
||||
@@ -76,9 +81,9 @@ class ExampleSecondaryTierManager(SecondaryTierManager):
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
True if the block is present, False if not found.
|
||||
HIT if the block is present, MISS if not found.
|
||||
"""
|
||||
return key in self.blocks
|
||||
return LookupResult.HIT if key in self.blocks else LookupResult.MISS
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
|
||||
@@ -24,7 +24,7 @@ from typing import TYPE_CHECKING
|
||||
from typing_extensions import override
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
@@ -137,8 +137,11 @@ class FileSystemTierManager(SecondaryTierManager):
|
||||
return RequestOffloadingContext()
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
return self._lookup_manager.lookup(key, req_context)
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
result = self._lookup_manager.lookup(key, req_context)
|
||||
if result is None:
|
||||
return LookupResult.RETRY
|
||||
return LookupResult.HIT if result else LookupResult.MISS
|
||||
|
||||
@override
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
|
||||
@@ -32,6 +32,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.offloading.metrics import (
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import (
|
||||
LoadStoreSpec,
|
||||
LookupResult,
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadKey,
|
||||
@@ -233,7 +234,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
)
|
||||
|
||||
@override
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
"""
|
||||
Check whether a single block is offloaded and ready.
|
||||
|
||||
@@ -248,33 +249,34 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
req_context: Per-request context.
|
||||
|
||||
Returns:
|
||||
True — block is ready in the primary tier.
|
||||
None — block found but not yet ready (primary in-flight,
|
||||
promotion started, or a secondary tier is busy).
|
||||
False — block not found in any tier, or primary is full
|
||||
and cannot accept a promotion.
|
||||
HIT — block is ready in the primary tier.
|
||||
HIT_PENDING — block found but not yet readable (write
|
||||
in-flight on the primary tier).
|
||||
RETRY — promotion started or a secondary tier is busy.
|
||||
MISS — block not found in any tier, or primary is full
|
||||
and cannot accept a promotion.
|
||||
"""
|
||||
self._maybe_process_finished_jobs()
|
||||
|
||||
primary_hit = self.primary_tier.lookup(key, req_context)
|
||||
if primary_hit is True:
|
||||
return True
|
||||
if primary_hit is None:
|
||||
return None
|
||||
if primary_hit is LookupResult.HIT:
|
||||
return LookupResult.HIT
|
||||
if primary_hit is LookupResult.HIT_PENDING:
|
||||
return LookupResult.HIT_PENDING
|
||||
|
||||
any_none = False
|
||||
any_retry = False
|
||||
for tier in self.secondary_tiers:
|
||||
result = tier.lookup(key, req_context)
|
||||
if result is True:
|
||||
if result is LookupResult.HIT:
|
||||
if not self._initiate_promotion(tier, key, req_context):
|
||||
return False # primary full, block unavailable
|
||||
return None # promotion started, retry later
|
||||
if result is None:
|
||||
any_none = True
|
||||
return LookupResult.MISS
|
||||
return LookupResult.RETRY
|
||||
if result is LookupResult.RETRY:
|
||||
any_retry = True
|
||||
|
||||
if any_none:
|
||||
return None
|
||||
return False
|
||||
if any_retry:
|
||||
return LookupResult.RETRY
|
||||
return LookupResult.MISS
|
||||
|
||||
def _initiate_promotion(
|
||||
self,
|
||||
@@ -467,7 +469,9 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
"""
|
||||
# Filter out keys that are not ready in primary (e.g. in-flight)
|
||||
ready_keys = tuple(
|
||||
k for k in keys if self.primary_tier.lookup(k, req_context) is True
|
||||
k
|
||||
for k in keys
|
||||
if self.primary_tier.lookup(k, req_context) is LookupResult.HIT
|
||||
)
|
||||
if not ready_keys:
|
||||
return
|
||||
|
||||
@@ -10,7 +10,7 @@ from typing import TYPE_CHECKING, NamedTuple
|
||||
from vllm.distributed.nixl_utils import NixlWrapper as nixl_agent
|
||||
from vllm.distributed.nixl_utils import nixl_agent_config
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.kv_offload.base import OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.base import LookupResult, OffloadKey, ReqContext
|
||||
from vllm.v1.kv_offload.file_mapper import FileMapper
|
||||
from vllm.v1.kv_offload.tiering.async_lookup import AsyncLookupManager
|
||||
from vllm.v1.kv_offload.tiering.base import (
|
||||
@@ -221,8 +221,11 @@ class ObjectStoreSecondaryTierManager(SecondaryTierManager):
|
||||
|
||||
self._transfers[job_id] = TransferEntry(xfer_handle, files_desc, obj_handle)
|
||||
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> bool | None:
|
||||
return self._lookup_manager.lookup(key, req_context)
|
||||
def lookup(self, key: OffloadKey, req_context: ReqContext) -> LookupResult:
|
||||
result = self._lookup_manager.lookup(key, req_context)
|
||||
if result is None:
|
||||
return LookupResult.RETRY
|
||||
return LookupResult.HIT if result else LookupResult.MISS
|
||||
|
||||
def submit_store(self, job_metadata: JobMetadata) -> None:
|
||||
obj_keys = (self._file_mapper.get_file_name(k) for k in job_metadata.keys)
|
||||
|
||||
Reference in New Issue
Block a user