[kv_offload+HMA][4/N]: Support sliding window lookup (#36645)
Signed-off-by: Or Ozeri <oro@il.ibm.com> Co-authored-by: Nicolò Lucchesi <nlucches@redhat.com>
This commit is contained in:
co-authored by
Nicolò Lucchesi
parent
2aab9acf48
commit
f774ba028a
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from itertools import islice
|
||||
from typing import Any, NamedTuple
|
||||
@@ -132,6 +132,49 @@ class OffloadingConnectorScheduler:
|
||||
self._reqs_being_stored = defaultdict[ReqId, set[OffloadKey]](set)
|
||||
self._reqs_being_loaded = defaultdict[ReqId, set[OffloadKey]](set)
|
||||
|
||||
def _maximal_prefix_lookup(
|
||||
self, keys: Iterable[OffloadKey], req_context: ReqContext
|
||||
) -> int | None:
|
||||
"""Find the length of the maximal prefix of offloaded blocks."""
|
||||
hit_count = 0
|
||||
defer_lookup = False
|
||||
for key in keys:
|
||||
result = self.manager.lookup(key, req_context)
|
||||
if result is None:
|
||||
defer_lookup = True
|
||||
# continue lookup to allow manager to kick-off async lookups
|
||||
# for all blocks (until a miss is detected)
|
||||
result = True
|
||||
if not result:
|
||||
break
|
||||
hit_count += 1
|
||||
return hit_count if not defer_lookup else None
|
||||
|
||||
def _sliding_window_lookup(
|
||||
self,
|
||||
keys: Sequence[OffloadKey],
|
||||
sliding_window_size: int,
|
||||
req_context: ReqContext,
|
||||
) -> int | None:
|
||||
"""Find the maximal ending position of consecutive offloaded blocks
|
||||
within a sliding window."""
|
||||
defer_lookup = False
|
||||
consecutive_hits = 0
|
||||
for idx in range(len(keys) - 1, -1, -1):
|
||||
result = self.manager.lookup(keys[idx], req_context)
|
||||
if result is None:
|
||||
defer_lookup = True
|
||||
# continue lookup to allow manager to kick-off async lookups
|
||||
# for all blocks (until a hit is detected)
|
||||
result = False
|
||||
if not result:
|
||||
consecutive_hits = 0
|
||||
else:
|
||||
consecutive_hits += 1
|
||||
if consecutive_hits == sliding_window_size:
|
||||
return idx + sliding_window_size if not defer_lookup else None
|
||||
return consecutive_hits if not defer_lookup else None
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
@@ -184,9 +227,10 @@ class OffloadingConnectorScheduler:
|
||||
return 0, False
|
||||
|
||||
start_block_idx = num_computed_tokens // group_config.offloaded_block_size
|
||||
hits = self.manager.lookup(
|
||||
offload_keys[start_block_idx:],
|
||||
req_status.req_context,
|
||||
# Full attention relays on all previous KV cache blocks.
|
||||
# Thus, we search for a maximal prefix of KV cache which are all cached.
|
||||
hits = self._maximal_prefix_lookup(
|
||||
offload_keys[start_block_idx:], req_status.req_context
|
||||
)
|
||||
if hits is None:
|
||||
# indicates a lookup that should be tried later
|
||||
|
||||
Reference in New Issue
Block a user