[BugFix][kv_offload]: Prevent offloading stale sliding window blocks (#42959)

Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Or Ozeri
2026-06-02 05:59:48 +03:00
committed by GitHub
parent 279d25f5cb
commit 480fadab1b
2 changed files with 156 additions and 40 deletions
@@ -1305,3 +1305,79 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
(1, 7),
),
)
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_stale_sliding_window_block_after_prepare_store_failure(
request_runner, async_scheduling: bool
):
"""Regression test: when prepare_store fails (returns None), offloading is
delayed. Meanwhile, sliding window blocks get freed and reallocated to the
same request. On retry, the stale block_id must be detected and skipped.
Without the fix, the stale block_id would either:
- Cause a KeyError in _remove_pending_job (duplicate in
_block_id_to_pending_jobs)
- Silently offload wrong data under a wrong key
"""
block_size = 4
# sliding_window = 8 -> window of 2 blocks
sliding_window = 8
# Use a tight GPU block budget so freed sliding window blocks are
# immediately reused by the same request's new allocations.
num_gpu_blocks = 4
kv_cache_groups = [
KVCacheGroupSpec(
["layer0"],
SlidingWindowSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
sliding_window=sliding_window,
),
),
]
runner = request_runner(
block_size=block_size,
num_gpu_blocks=num_gpu_blocks,
async_scheduling=async_scheduling,
kv_cache_groups=kv_cache_groups,
)
# Request with 3 blocks of prompt. Window = 2 blocks, so block 0 is
# outside the window but won't be freed until the next allocate_slots.
runner.new_request(token_ids=[0] * block_size * 3)
# First step: prepare_store FAILS -> offloading delayed.
# next_stored_block_idx stays at 0, block_ids[0] still holds the
# original block_id for position 0.
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
runner.run(decoded_tokens=[0])
runner.manager.prepare_store.assert_called()
# Second step: decode more tokens -> block 3 allocated.
# allocate_slots calls remove_skipped_blocks which frees block 0
# (it's now outside the sliding window). With num_gpu_blocks=4,
# the freed block is immediately reused for the new allocation.
# prepare_store still fails so offloading is still delayed.
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
runner.run(decoded_tokens=[0] * block_size)
# Now prepare_store succeeds.
# Without the fix, the request would try to offload the stale block_id
# at position 0 (now reused at position 3), causing a duplicate in
# sliding_window_block_ids and eventually a KeyError.
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
# block_ids=[0, ?, 3, 1]: positions 0 and 1 are zeroed (stale blocks that
# were freed by the sliding window and reallocated). Only blocks at
# positions 2 and 3 (request offsets 2, 3) are stored.
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=(2, 3),
expected_flushed=(2, 3) if not async_scheduling else (),
)
@@ -286,6 +286,8 @@ class OffloadingConnectorScheduler:
self._req_status: dict[ReqId, RequestOffloadState] = {}
self._current_batch_load_jobs: dict[int, TransferJob] = {}
self._current_batch_jobs_to_flush: set[int] = set()
# GPU block IDs allocated in the current engine step
self._current_batch_allocated_block_ids: set[int] = set()
# if GPU prefix caching is enabled,
# track loaded blocks to avoid redundant loads
self._blocks_being_loaded: set[OffloadKey] | None = (
@@ -589,6 +591,10 @@ class OffloadingConnectorScheduler:
req_status.group_states,
blocks.blocks,
):
self._current_batch_allocated_block_ids.update(
block.block_id for block in group_blocks if block.block_id != 0
)
gpu_block_size = group_config.gpu_block_size
offloaded_block_size = group_config.offloaded_block_size
offload_keys = group_state.offload_keys
@@ -638,17 +644,6 @@ class OffloadingConnectorScheduler:
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
group_state.next_stored_block_idx = num_blocks
# Fence dst blocks against finished-request pending stores.
if (
self._block_id_to_pending_jobs
and not self._block_id_to_pending_jobs.keys().isdisjoint(dst_block_ids)
):
self._current_batch_jobs_to_flush.update(
jid
for bid in dst_block_ids
for jid in self._block_id_to_pending_jobs.get(bid, ())
)
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
dst_spec = GPULoadStoreSpec(
dst_block_ids, group_sizes=group_sizes, block_indices=block_indices
@@ -672,37 +667,68 @@ class OffloadingConnectorScheduler:
if self._blocks_being_loaded is not None:
self._blocks_being_loaded.update(keys_to_load)
def _build_store_jobs(
self,
scheduler_output: SchedulerOutput,
) -> dict[int, TransferJob]:
block_size_factor = self.config.block_size_factor
store_jobs: dict[int, TransferJob] = {}
# iterate over both new and cached requests
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
"""
Update request states from the Scheduler's output.
"""
# new_block_ids_end[req_id][i] = end of pre-existing block_ids for
# the i-th sliding window group (before this step's extend).
# Used to detect sliding window blocks that got re-allocated.
new_block_ids_end: dict[str, tuple[int, ...]] = {}
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
req_status = self._req_status[req_id]
req_status.update_offload_keys()
req = req_status.req
if preempted:
for group_state in req_status.group_states:
group_state.block_ids.clear()
if new_block_id_groups:
if self._sliding_window_groups:
new_block_ids_end[req_id] = tuple(
len(req_status.group_states[grp_idx].block_ids)
for grp_idx in self._sliding_window_groups
)
req_status.update_block_id_groups(new_block_id_groups)
# Fence new blocks against in-flight stores.
if self._block_id_to_pending_jobs:
new_blocks_flat = [
bid for new_blocks in new_block_id_groups for bid in new_blocks
]
if not self._block_id_to_pending_jobs.keys().isdisjoint(
new_blocks_flat
):
self._current_batch_jobs_to_flush.update(
jid
for bid in new_blocks_flat
for jid in self._block_id_to_pending_jobs.get(bid, ())
)
for new_blocks in new_block_id_groups:
for bid in new_blocks:
if bid != 0:
self._current_batch_allocated_block_ids.add(bid)
# Zero out stale block_ids in sliding window groups' pending-store
# positions. Only sliding window groups can have stale entries (blocks
# freed by remove_skipped_blocks then reallocated). Only positions in
# [next_stored_block_idx * bsf, end) need checking where end is the
# pre-extend length: earlier positions were already offloaded, later
# ones are fresh allocations from this step.
if self._sliding_window_groups and self._current_batch_allocated_block_ids:
block_size_factor = self.config.block_size_factor
for req_id, req_status in self._req_status.items():
ends = new_block_ids_end.get(req_id)
for i, grp_idx in enumerate(self._sliding_window_groups):
group_state = req_status.group_states[grp_idx]
start = group_state.next_stored_block_idx * block_size_factor
end = ends[i] if ends is not None else len(group_state.block_ids)
for j in range(start, end):
if (
group_state.block_ids[j]
in self._current_batch_allocated_block_ids
):
group_state.block_ids[j] = 0
def _build_store_jobs(
self,
scheduler_output: SchedulerOutput,
) -> dict[int, TransferJob]:
block_size_factor = self.config.block_size_factor
store_jobs: dict[int, TransferJob] = {}
for req_id in scheduler_output.num_scheduled_tokens:
req_status = self._req_status.get(req_id)
if req_status is None:
continue
req = req_status.req
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
@@ -735,11 +761,8 @@ class OffloadingConnectorScheduler:
# For each block to offload, take the last corresponding GPU block.
# e.g. if block size factor is 3 and GPU block IDs are
# 1 5 6 7 2 4 9 3 8 then we'll take blocks 6 4 8.
# We will use these GPU blocks to determine if the block needs
# offloading, or (if the GPU block ID is 0) this block should
# be skipped due to sliding window attention / SSM.
# We know that if a block is skipped, then all the previous blocks
# are skipped as well. This is why we take the last of each block.
# A block_id of 0 means either a sliding window / SSM skip
# or a stale entry that was zeroed out — skip it either way.
offload_block_ids = group_state.block_ids[
start_block_idx * block_size_factor
+ block_size_factor
@@ -814,10 +837,8 @@ class OffloadingConnectorScheduler:
for i in range(block_size_factor):
block_id = block_ids[gpu_block_idx + i]
if block_id == 0:
# skipped blocks cannot appear after non-skipped blocks
assert start_gpu_block_idx is None
continue
elif start_gpu_block_idx is None:
if start_gpu_block_idx is None:
start_gpu_block_idx = gpu_block_idx + i
src_block_ids.append(block_id)
num_group_blocks += 1
@@ -875,6 +896,9 @@ class OffloadingConnectorScheduler:
def build_connector_meta(
self, scheduler_output: SchedulerOutput
) -> KVConnectorMetadata:
self._update_req_states(scheduler_output)
# Flush jobs for preempted requests.
for req_id in scheduler_output.preempted_req_ids or ():
req_status = self._req_status.get(req_id)
if req_status is None or not req_status.transfer_jobs:
@@ -883,6 +907,20 @@ class OffloadingConnectorScheduler:
assert self._jobs[any_jid].is_store
self._current_batch_jobs_to_flush.update(req_status.transfer_jobs)
# Flush jobs that contain re-allocated blocks.
if (
self._block_id_to_pending_jobs
and not self._block_id_to_pending_jobs.keys().isdisjoint(
self._current_batch_allocated_block_ids
)
):
self._current_batch_jobs_to_flush.update(
jid
for bid in self._current_batch_allocated_block_ids
if bid in self._block_id_to_pending_jobs
for jid in self._block_id_to_pending_jobs[bid]
)
# If all tracked requests are finished, flush all pending jobs
# (both store and load) - there might not be a future scheduler
# step to trigger their completion.
@@ -898,6 +936,7 @@ class OffloadingConnectorScheduler:
)
self._current_batch_load_jobs = {}
self._current_batch_jobs_to_flush = set()
self._current_batch_allocated_block_ids = set()
return meta
def update_connector_output(self, connector_output: KVConnectorOutput):
@@ -1013,6 +1052,7 @@ class OffloadingConnectorScheduler:
# reset_cache cannot be called in the middle of a schedule step
assert not self._current_batch_load_jobs
assert not self._current_batch_jobs_to_flush
assert not self._current_batch_allocated_block_ids
# Flush all in-flight jobs
self._current_batch_jobs_to_flush.update(self._jobs.keys())