[kv_offload+HMA][12/N]: Scheduler-side support for sliding window groups (#41228)

Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Or Ozeri
2026-05-01 06:59:17 +03:00
committed by GitHub
parent 6b6ac6c3c7
commit 941fb50835
@@ -18,6 +18,12 @@ from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheSpec,
MambaSpec,
SlidingWindowSpec,
)
from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
OffloadingManager,
@@ -44,8 +50,12 @@ class TransferJobStatus:
# Offload keys this job covers; passed to manager.complete_*().
keys: set[OffloadKey]
is_store: bool
# GPU blocks the fence tracks. Store src blocks; None for loads.
gpu_block_ids: list[int] | None = None
# Store src block IDs whose ref_cnt protects them while the request
# runs. Only registered in _block_id_to_pending_jobs on request_finished.
non_sliding_window_block_ids: list[int] | None = None
# Store src block IDs that may be freed before the request finishes.
# Registered in _block_id_to_pending_jobs at store creation time.
sliding_window_block_ids: list[int] | None = None
class GroupOffloadConfig(NamedTuple):
@@ -53,6 +63,23 @@ class GroupOffloadConfig(NamedTuple):
gpu_block_size: int
offloaded_block_size: int
hash_block_size_factor: int
# None below means full attention
sliding_window_size_in_blocks: int | None
def get_sliding_window_size_in_blocks(
kv_cache_spec: KVCacheSpec, offloaded_block_size: int
) -> int | None:
if isinstance(kv_cache_spec, SlidingWindowSpec):
assert kv_cache_spec.sliding_window > 0
return cdiv(kv_cache_spec.sliding_window, offloaded_block_size)
if isinstance(kv_cache_spec, MambaSpec):
# Mamba depends on a single state
return 1
assert isinstance(kv_cache_spec, FullAttentionSpec)
return None
class SchedulerOffloadConfig(NamedTuple):
@@ -73,6 +100,10 @@ class SchedulerOffloadConfig(NamedTuple):
(gpu_block_size * spec.block_size_factor)
// spec.hash_block_size
),
sliding_window_size_in_blocks=get_sliding_window_size_in_blocks(
spec.kv_cache_config.kv_cache_groups[idx].kv_cache_spec,
gpu_block_size * spec.block_size_factor,
),
)
for idx, gpu_block_size in enumerate(spec.gpu_block_size)
),
@@ -86,6 +117,9 @@ class RequestGroupState:
block_ids: list[int] = field(default_factory=list)
# index of next block (of size offloaded_block_size) to offload
next_stored_block_idx: int = 0
# number of offloaded blocks hit (including GPU prefix cache)
# when the request first started
num_hit_blocks: int = 0
@dataclass(slots=True)
@@ -139,6 +173,14 @@ class RequestOffloadState:
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
group_state.next_stored_block_idx = num_blocks
def update_num_hit_blocks(self, num_cached_tokens: int) -> None:
for group_config, group_state in zip(
self.config.kv_group_configs, self.group_states
):
group_state.num_hit_blocks = (
num_cached_tokens // group_config.offloaded_block_size
)
class OffloadingConnectorScheduler:
"""Implementation of Scheduler side methods"""
@@ -147,12 +189,25 @@ class OffloadingConnectorScheduler:
self.config = SchedulerOffloadConfig.from_spec(spec)
self.manager: OffloadingManager = spec.get_manager()
attention_groups: list[int] = []
for idx, _ in enumerate(spec.kv_cache_config.kv_cache_groups):
# currently treat all groups as full attention
attention_groups.append(idx)
full_attention_groups: list[int] = []
sliding_window_groups: list[int] = []
for group_config in self.config.kv_group_configs:
if group_config.sliding_window_size_in_blocks is None:
full_attention_groups.append(group_config.group_idx)
else:
sliding_window_groups.append(group_config.group_idx)
self.lookup_groups = attention_groups
# sort sliding window groups by window size in decreasing order
def _sliding_window_sort_key(i: int) -> int:
val = self.config.kv_group_configs[i].sliding_window_size_in_blocks
assert val is not None
return val
sliding_window_groups.sort(key=_sliding_window_sort_key, reverse=True)
# used by _lookup
self._sliding_window_groups: tuple[int, ...] = tuple(sliding_window_groups)
self._lookup_groups = tuple(full_attention_groups) + self._sliding_window_groups
self._req_status: dict[ReqId, RequestOffloadState] = {}
self._current_batch_load_jobs: dict[int, TransferJob] = {}
@@ -167,8 +222,11 @@ class OffloadingConnectorScheduler:
self._job_counter: int = 0
self._jobs: dict[int, TransferJobStatus] = {}
# block_id -> pending store job_ids. Populated only for finished
# requests (running-request blocks are protected by their ref_cnt).
# block_id -> pending store job_ids. Used to track jobs that needs
# flushing in case a block is re-allocated by the KV cache manager.
# Populated only for finished requests (running-request blocks are
# protected by their ref_cnt) and for sliding window blocks (which can
# be freed before a request finishes).
self._block_id_to_pending_jobs: dict[int, set[int]] = {}
def _generate_job_id(self) -> int:
@@ -176,10 +234,18 @@ class OffloadingConnectorScheduler:
self._job_counter += 1
return job_id
def _remove_pending_job(self, job_id: int, block_ids: list[int] | None) -> None:
for bid in block_ids or ():
pending = self._block_id_to_pending_jobs[bid]
pending.remove(job_id)
if not pending:
del self._block_id_to_pending_jobs[bid]
def _maximal_prefix_lookup(
self, keys: Iterable[OffloadKey], req_context: ReqContext
) -> int | None:
"""Find the length of the maximal prefix of offloaded blocks."""
"""Return the number of consecutive offloaded blocks from the start,
or None if the backend deferred a lookup."""
hit_count = 0
defer_lookup = False
for key in keys:
@@ -200,8 +266,9 @@ class OffloadingConnectorScheduler:
sliding_window_size: int,
req_context: ReqContext,
) -> int | None:
"""Find the maximal ending position of consecutive offloaded blocks
within a sliding window."""
"""Return the end index (in `keys`) of the last run of
`sliding_window_size` consecutive hits, scanning from the end.
Returns 0 on miss, None if the backend deferred a lookup."""
defer_lookup = False
consecutive_hits = 0
for idx in range(len(keys) - 1, -1, -1):
@@ -219,6 +286,160 @@ class OffloadingConnectorScheduler:
return idx + sliding_window_size if not defer_lookup else None
return consecutive_hits if not defer_lookup else None
def _touch(self, req_status: RequestOffloadState):
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
if group_config.sliding_window_size_in_blocks is None:
self.manager.touch(group_state.offload_keys)
else:
# we aim to keep just blocks that are necessary to hit
# the original request (+ decoded blocks)
blocks_to_skip = max(
0,
group_state.num_hit_blocks
- group_config.sliding_window_size_in_blocks,
)
self.manager.touch(group_state.offload_keys[blocks_to_skip:])
def _lookup(self, req_status: RequestOffloadState) -> int | None:
"""
Find how many tokens beyond num_locally_computed_tokens can be loaded.
Iterates full-attention groups first (prefix lookup), then sliding-window
groups (suffix lookup). Each group may tighten max_hit_size_tokens, which
can invalidate an earlier group's result, so the loop re-runs when that
happens until num_hit_tokens converges.
"""
num_computed_tokens = req_status.num_locally_computed_tokens
max_hit_size_tokens: int = req_status.req.num_tokens
if self._sliding_window_groups:
# the last prompt token has to be recomputed to get the logprobs
# for sliding window attention, we must reduce by 1 to make sure
# we still have a hit after reduction
max_hit_size_tokens -= 1
num_hit_tokens: int = 0
defer_lookup = False
lookup_groups = self._lookup_groups
while lookup_groups:
looked_up_sliding_window: bool = False
groups_iter = iter(lookup_groups)
lookup_groups = ()
for group_idx in groups_iter:
group_config: GroupOffloadConfig = self.config.kv_group_configs[
group_idx
]
group_state: RequestGroupState = req_status.group_states[group_idx]
offloaded_block_size = group_config.offloaded_block_size
offload_keys = group_state.offload_keys
assert (
len(offload_keys)
>= req_status.req.num_tokens // offloaded_block_size
)
# Constrain to block-aligned boundary for this group
max_hit_size_tokens = min(
max_hit_size_tokens, len(offload_keys) * offloaded_block_size
)
if max_hit_size_tokens - num_computed_tokens < offloaded_block_size:
# we can only load less than a block, better skip
return 0
num_blocks = min(
cdiv(max_hit_size_tokens, offloaded_block_size), len(offload_keys)
)
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
sliding_window_size_in_blocks = (
group_config.sliding_window_size_in_blocks
)
# end index (in the sliced offload_keys) up to which we
# have backend-confirmed hits
num_hit_blocks: int | None
if sliding_window_size_in_blocks is None:
num_hit_blocks = self._maximal_prefix_lookup(
offload_keys, req_status.req_context
)
else:
num_hit_blocks = self._sliding_window_lookup(
offload_keys,
sliding_window_size_in_blocks,
req_status.req_context,
)
if num_hit_blocks == 0:
return 0
if num_hit_blocks is None:
defer_lookup = True
else:
max_hit_size_tokens = min(
max_hit_size_tokens,
offloaded_block_size * (start_block_idx + num_hit_blocks),
)
new_num_hit_tokens = max_hit_size_tokens - num_computed_tokens
if new_num_hit_tokens < offloaded_block_size:
# we can only load less than a block, better skip
return 0
if new_num_hit_tokens < num_hit_tokens:
if defer_lookup:
# make another iteration on all groups to check
# if we still need to defer lookup
defer_lookup = False
lookup_groups = self._lookup_groups
elif looked_up_sliding_window and not lookup_groups:
# we need another iteration to confirm previously looked up
# sliding window works with the new_num_hit_tokens
lookup_groups = self._sliding_window_groups
looked_up_sliding_window |= sliding_window_size_in_blocks is not None
num_hit_tokens = new_num_hit_tokens
if defer_lookup:
logger.debug(
"Offloading manager delayed request %s as backend requested",
req_status.req.request_id,
)
return None
# possibly delay request if any of the hit blocks is already being loaded
if self._blocks_being_loaded:
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
offloaded_block_size = group_config.offloaded_block_size
sliding_window_size_in_blocks = (
group_config.sliding_window_size_in_blocks
)
offload_keys = group_state.offload_keys
num_blocks = cdiv(
num_computed_tokens + num_hit_tokens, offloaded_block_size
)
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
if sliding_window_size_in_blocks is not None:
offload_keys = offload_keys[-sliding_window_size_in_blocks:]
if any(key in self._blocks_being_loaded for key in offload_keys):
# hit blocks are being loaded, delay request
logger.debug(
"Delaying request %s since some of its"
" blocks are already being loaded",
req_status.req.request_id,
)
return None
logger.debug(
"Request %s hit %s offloaded tokens after %s GPU hit tokens",
req_status.req.request_id,
num_hit_tokens,
num_computed_tokens,
)
return num_hit_tokens
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
@@ -241,96 +462,28 @@ class OffloadingConnectorScheduler:
- `True` if tokens will be loaded asynchronously
(between scheduler steps).
"""
is_new_request = False
if req_status := self._req_status.get(request.request_id):
# make sure block IDs are cleared
for group_state in req_status.group_states:
group_state.block_ids.clear()
else:
is_new_request = True
req_status = RequestOffloadState(config=self.config, req=request)
self._req_status[request.request_id] = req_status
req_status.update_offload_keys()
req_status.num_locally_computed_tokens = num_computed_tokens
for gs in req_status.group_states:
self.manager.touch(gs.offload_keys)
# Start with the full request size as the maximum loadable
max_hit_size_tokens: int = req_status.req.num_tokens
num_hit_tokens: int = 0
defer_lookup = False
delay_request = False
for group_idx in self.lookup_groups:
group_config: GroupOffloadConfig = self.config.kv_group_configs[group_idx]
offloaded_block_size = group_config.offloaded_block_size
offload_keys = req_status.group_states[group_idx].offload_keys
num_blocks = max_hit_size_tokens // offloaded_block_size
assert len(offload_keys) >= num_blocks
# Constrain to block-aligned boundary for this group
max_hit_size_tokens = num_blocks * offloaded_block_size
num_hit_tokens = max_hit_size_tokens - num_computed_tokens
if num_hit_tokens < offloaded_block_size:
# we can only load less than a block, better skip
return 0, False
start_block_idx = num_computed_tokens // offloaded_block_size
offload_keys = offload_keys[start_block_idx:num_blocks]
# Full attention relies on all previous KV cache blocks.
# Thus, we search for a maximal prefix of KV cache which are all cached.
block_hits = self._maximal_prefix_lookup(
offload_keys, req_status.req_context
num_hit_tokens = self._lookup(req_status)
if is_new_request:
req_status.update_num_hit_blocks(
num_computed_tokens + (num_hit_tokens or 0)
)
if block_hits == 0:
return 0, False
if block_hits is None:
defer_lookup = True
else:
# Further constrain based on what's actually available by backend
max_hit_size_tokens = offloaded_block_size * (
start_block_idx + block_hits
)
self._touch(req_status)
num_hit_tokens = max_hit_size_tokens - num_computed_tokens
if num_hit_tokens < offloaded_block_size:
# we can only load less than a block, better skip
return 0, False
if (
block_hits
and self._blocks_being_loaded
and any(
key in self._blocks_being_loaded
for key in offload_keys[:block_hits]
)
):
# hit blocks are being loaded, delay request
delay_request = True
if defer_lookup:
logger.debug(
"Offloading manager delayed request %s as backend requested",
req_status.req.request_id,
)
return None, False
if delay_request:
logger.debug(
"Delaying request %s since some of its blocks are already being loaded",
req_status.req.request_id,
)
return None, False
logger.debug(
"Request %s hit %s offloaded tokens after %s GPU hit tokens",
request.request_id,
num_hit_tokens,
num_computed_tokens,
)
return num_hit_tokens, True
return num_hit_tokens, bool(num_hit_tokens)
def update_state_after_alloc(
self, request: Request, blocks: KVCacheBlocks, num_external_tokens: int
@@ -375,6 +528,13 @@ class OffloadingConnectorScheduler:
)
num_pending_gpu_blocks = num_gpu_blocks - num_locally_computed_gpu_blocks
if group_config.sliding_window_size_in_blocks is not None:
assert (
num_pending_gpu_blocks
<= group_config.sliding_window_size_in_blocks
* self.config.block_size_factor
)
num_blocks = cdiv(num_cached_tokens, offloaded_block_size)
assert len(offload_keys) >= num_blocks
if num_pending_gpu_blocks:
@@ -513,17 +673,21 @@ class OffloadingConnectorScheduler:
req_status.advance_stored_idx(num_offloadable_tokens)
continue
for group_state in req_status.group_states:
self.manager.touch(group_state.offload_keys)
self._touch(req_status)
keys_to_store = set(store_output.keys_to_store)
group_sizes: list[int] = []
block_indices: list[int] = []
src_block_ids: list[int] = []
sliding_window_block_ids: list[int] = []
non_sliding_window_block_ids: list[int] = []
for group_config, group_state in zip(
self.config.kv_group_configs, req_status.group_states
):
is_sliding_window = (
group_config.sliding_window_size_in_blocks is not None
)
num_blocks = num_offloadable_tokens // group_config.offloaded_block_size
start_block_idx = group_state.next_stored_block_idx
block_ids = group_state.block_ids
@@ -547,6 +711,11 @@ class OffloadingConnectorScheduler:
elif start_gpu_block_idx is None:
start_gpu_block_idx = gpu_block_idx + i
src_block_ids.append(block_id)
if is_sliding_window:
sliding_window_block_ids.append(block_id)
else:
non_sliding_window_block_ids.append(block_id)
group_sizes.append(num_group_blocks)
block_indices.append(start_gpu_block_idx or 0)
group_state.next_stored_block_idx = num_blocks
@@ -562,12 +731,21 @@ class OffloadingConnectorScheduler:
any_jid = next(iter(req_status.transfer_jobs))
assert self._jobs[any_jid].is_store
req_status.transfer_jobs.add(job_id)
# Watch sliding window blocks as they may get evicted
# before the request finishes
for bid in sliding_window_block_ids or ():
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
# the non-sliding window blocks will be watched only
# when the request finishes
self._jobs[job_id] = TransferJobStatus(
req_id=req_id,
pending_count=self.config.num_workers,
keys=set(keys_to_store),
is_store=True,
gpu_block_ids=src_block_ids,
non_sliding_window_block_ids=non_sliding_window_block_ids,
sliding_window_block_ids=sliding_window_block_ids or None,
)
store_jobs[job_id] = TransferJob(
@@ -632,12 +810,16 @@ class OffloadingConnectorScheduler:
self._blocks_being_loaded.difference_update(job_status.keys)
req_status = self._req_status[job_status.req_id]
if self._block_id_to_pending_jobs and req_status.req.is_finished():
for bid in job_status.gpu_block_ids or ():
pending = self._block_id_to_pending_jobs[bid]
pending.remove(job_id)
if not pending:
del self._block_id_to_pending_jobs[bid]
if self._block_id_to_pending_jobs:
# Sliding window blocks are tracked from store creation
# and must be cleaned up unconditionally.
self._remove_pending_job(job_id, job_status.sliding_window_block_ids)
# Non-sliding-window blocks are only tracked after
# request_finished, so only clean up for finished requests.
if req_status.req.is_finished():
self._remove_pending_job(
job_id, job_status.non_sliding_window_block_ids
)
del self._jobs[job_id]
req_status.transfer_jobs.remove(job_id)
@@ -671,7 +853,7 @@ class OffloadingConnectorScheduler:
# Register them so future block reuse triggers a flush.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.gpu_block_ids or ():
for bid in job_status.non_sliding_window_block_ids or ():
self._block_id_to_pending_jobs.setdefault(bid, set()).add(job_id)
return False, None