[BugFix][kv_offload]: Prevent offloading stale sliding window blocks (#42959)
Signed-off-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -1305,3 +1305,79 @@ def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
(1, 7),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_stale_sliding_window_block_after_prepare_store_failure(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""Regression test: when prepare_store fails (returns None), offloading is
|
||||
delayed. Meanwhile, sliding window blocks get freed and reallocated to the
|
||||
same request. On retry, the stale block_id must be detected and skipped.
|
||||
|
||||
Without the fix, the stale block_id would either:
|
||||
- Cause a KeyError in _remove_pending_job (duplicate in
|
||||
_block_id_to_pending_jobs)
|
||||
- Silently offload wrong data under a wrong key
|
||||
"""
|
||||
block_size = 4
|
||||
# sliding_window = 8 -> window of 2 blocks
|
||||
sliding_window = 8
|
||||
# Use a tight GPU block budget so freed sliding window blocks are
|
||||
# immediately reused by the same request's new allocations.
|
||||
num_gpu_blocks = 4
|
||||
|
||||
kv_cache_groups = [
|
||||
KVCacheGroupSpec(
|
||||
["layer0"],
|
||||
SlidingWindowSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
sliding_window=sliding_window,
|
||||
),
|
||||
),
|
||||
]
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=num_gpu_blocks,
|
||||
async_scheduling=async_scheduling,
|
||||
kv_cache_groups=kv_cache_groups,
|
||||
)
|
||||
|
||||
# Request with 3 blocks of prompt. Window = 2 blocks, so block 0 is
|
||||
# outside the window but won't be freed until the next allocate_slots.
|
||||
runner.new_request(token_ids=[0] * block_size * 3)
|
||||
|
||||
# First step: prepare_store FAILS -> offloading delayed.
|
||||
# next_stored_block_idx stays at 0, block_ids[0] still holds the
|
||||
# original block_id for position 0.
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
|
||||
runner.run(decoded_tokens=[0])
|
||||
runner.manager.prepare_store.assert_called()
|
||||
|
||||
# Second step: decode more tokens -> block 3 allocated.
|
||||
# allocate_slots calls remove_skipped_blocks which frees block 0
|
||||
# (it's now outside the sliding window). With num_gpu_blocks=4,
|
||||
# the freed block is immediately reused for the new allocation.
|
||||
# prepare_store still fails so offloading is still delayed.
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: None
|
||||
runner.run(decoded_tokens=[0] * block_size)
|
||||
|
||||
# Now prepare_store succeeds.
|
||||
# Without the fix, the request would try to offload the stale block_id
|
||||
# at position 0 (now reused at position 3), causing a duplicate in
|
||||
# sliding_window_block_ids and eventually a KeyError.
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
# block_ids=[0, ?, 3, 1]: positions 0 and 1 are zeroed (stale blocks that
|
||||
# were freed by the sliding window and reallocated). Only blocks at
|
||||
# positions 2 and 3 (request offsets 2, 3) are stored.
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=(2, 3),
|
||||
expected_flushed=(2, 3) if not async_scheduling else (),
|
||||
)
|
||||
|
||||
@@ -286,6 +286,8 @@ class OffloadingConnectorScheduler:
|
||||
self._req_status: dict[ReqId, RequestOffloadState] = {}
|
||||
self._current_batch_load_jobs: dict[int, TransferJob] = {}
|
||||
self._current_batch_jobs_to_flush: set[int] = set()
|
||||
# GPU block IDs allocated in the current engine step
|
||||
self._current_batch_allocated_block_ids: set[int] = set()
|
||||
# if GPU prefix caching is enabled,
|
||||
# track loaded blocks to avoid redundant loads
|
||||
self._blocks_being_loaded: set[OffloadKey] | None = (
|
||||
@@ -589,6 +591,10 @@ class OffloadingConnectorScheduler:
|
||||
req_status.group_states,
|
||||
blocks.blocks,
|
||||
):
|
||||
self._current_batch_allocated_block_ids.update(
|
||||
block.block_id for block in group_blocks if block.block_id != 0
|
||||
)
|
||||
|
||||
gpu_block_size = group_config.gpu_block_size
|
||||
offloaded_block_size = group_config.offloaded_block_size
|
||||
offload_keys = group_state.offload_keys
|
||||
@@ -638,17 +644,6 @@ class OffloadingConnectorScheduler:
|
||||
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
|
||||
group_state.next_stored_block_idx = num_blocks
|
||||
|
||||
# Fence dst blocks against finished-request pending stores.
|
||||
if (
|
||||
self._block_id_to_pending_jobs
|
||||
and not self._block_id_to_pending_jobs.keys().isdisjoint(dst_block_ids)
|
||||
):
|
||||
self._current_batch_jobs_to_flush.update(
|
||||
jid
|
||||
for bid in dst_block_ids
|
||||
for jid in self._block_id_to_pending_jobs.get(bid, ())
|
||||
)
|
||||
|
||||
src_spec = self.manager.prepare_load(keys_to_load, req_status.req_context)
|
||||
dst_spec = GPULoadStoreSpec(
|
||||
dst_block_ids, group_sizes=group_sizes, block_indices=block_indices
|
||||
@@ -672,37 +667,68 @@ class OffloadingConnectorScheduler:
|
||||
if self._blocks_being_loaded is not None:
|
||||
self._blocks_being_loaded.update(keys_to_load)
|
||||
|
||||
def _build_store_jobs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> dict[int, TransferJob]:
|
||||
block_size_factor = self.config.block_size_factor
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
# iterate over both new and cached requests
|
||||
def _update_req_states(self, scheduler_output: SchedulerOutput) -> None:
|
||||
"""
|
||||
Update request states from the Scheduler's output.
|
||||
"""
|
||||
|
||||
# new_block_ids_end[req_id][i] = end of pre-existing block_ids for
|
||||
# the i-th sliding window group (before this step's extend).
|
||||
# Used to detect sliding window blocks that got re-allocated.
|
||||
new_block_ids_end: dict[str, tuple[int, ...]] = {}
|
||||
|
||||
for req_id, new_block_id_groups, preempted in yield_req_data(scheduler_output):
|
||||
req_status = self._req_status[req_id]
|
||||
req_status.update_offload_keys()
|
||||
req = req_status.req
|
||||
|
||||
if preempted:
|
||||
for group_state in req_status.group_states:
|
||||
group_state.block_ids.clear()
|
||||
|
||||
if new_block_id_groups:
|
||||
if self._sliding_window_groups:
|
||||
new_block_ids_end[req_id] = tuple(
|
||||
len(req_status.group_states[grp_idx].block_ids)
|
||||
for grp_idx in self._sliding_window_groups
|
||||
)
|
||||
req_status.update_block_id_groups(new_block_id_groups)
|
||||
# Fence new blocks against in-flight stores.
|
||||
if self._block_id_to_pending_jobs:
|
||||
new_blocks_flat = [
|
||||
bid for new_blocks in new_block_id_groups for bid in new_blocks
|
||||
]
|
||||
if not self._block_id_to_pending_jobs.keys().isdisjoint(
|
||||
new_blocks_flat
|
||||
):
|
||||
self._current_batch_jobs_to_flush.update(
|
||||
jid
|
||||
for bid in new_blocks_flat
|
||||
for jid in self._block_id_to_pending_jobs.get(bid, ())
|
||||
)
|
||||
for new_blocks in new_block_id_groups:
|
||||
for bid in new_blocks:
|
||||
if bid != 0:
|
||||
self._current_batch_allocated_block_ids.add(bid)
|
||||
|
||||
# Zero out stale block_ids in sliding window groups' pending-store
|
||||
# positions. Only sliding window groups can have stale entries (blocks
|
||||
# freed by remove_skipped_blocks then reallocated). Only positions in
|
||||
# [next_stored_block_idx * bsf, end) need checking where end is the
|
||||
# pre-extend length: earlier positions were already offloaded, later
|
||||
# ones are fresh allocations from this step.
|
||||
if self._sliding_window_groups and self._current_batch_allocated_block_ids:
|
||||
block_size_factor = self.config.block_size_factor
|
||||
for req_id, req_status in self._req_status.items():
|
||||
ends = new_block_ids_end.get(req_id)
|
||||
for i, grp_idx in enumerate(self._sliding_window_groups):
|
||||
group_state = req_status.group_states[grp_idx]
|
||||
start = group_state.next_stored_block_idx * block_size_factor
|
||||
end = ends[i] if ends is not None else len(group_state.block_ids)
|
||||
for j in range(start, end):
|
||||
if (
|
||||
group_state.block_ids[j]
|
||||
in self._current_batch_allocated_block_ids
|
||||
):
|
||||
group_state.block_ids[j] = 0
|
||||
|
||||
def _build_store_jobs(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
) -> dict[int, TransferJob]:
|
||||
block_size_factor = self.config.block_size_factor
|
||||
store_jobs: dict[int, TransferJob] = {}
|
||||
for req_id in scheduler_output.num_scheduled_tokens:
|
||||
req_status = self._req_status.get(req_id)
|
||||
if req_status is None:
|
||||
continue
|
||||
req = req_status.req
|
||||
|
||||
num_scheduled_tokens = scheduler_output.num_scheduled_tokens[req_id]
|
||||
num_tokens_after_batch = req.num_computed_tokens + num_scheduled_tokens
|
||||
@@ -735,11 +761,8 @@ class OffloadingConnectorScheduler:
|
||||
# For each block to offload, take the last corresponding GPU block.
|
||||
# e.g. if block size factor is 3 and GPU block IDs are
|
||||
# 1 5 6 7 2 4 9 3 8 then we'll take blocks 6 4 8.
|
||||
# We will use these GPU blocks to determine if the block needs
|
||||
# offloading, or (if the GPU block ID is 0) this block should
|
||||
# be skipped due to sliding window attention / SSM.
|
||||
# We know that if a block is skipped, then all the previous blocks
|
||||
# are skipped as well. This is why we take the last of each block.
|
||||
# A block_id of 0 means either a sliding window / SSM skip
|
||||
# or a stale entry that was zeroed out — skip it either way.
|
||||
offload_block_ids = group_state.block_ids[
|
||||
start_block_idx * block_size_factor
|
||||
+ block_size_factor
|
||||
@@ -814,10 +837,8 @@ class OffloadingConnectorScheduler:
|
||||
for i in range(block_size_factor):
|
||||
block_id = block_ids[gpu_block_idx + i]
|
||||
if block_id == 0:
|
||||
# skipped blocks cannot appear after non-skipped blocks
|
||||
assert start_gpu_block_idx is None
|
||||
continue
|
||||
elif start_gpu_block_idx is None:
|
||||
if start_gpu_block_idx is None:
|
||||
start_gpu_block_idx = gpu_block_idx + i
|
||||
src_block_ids.append(block_id)
|
||||
num_group_blocks += 1
|
||||
@@ -875,6 +896,9 @@ class OffloadingConnectorScheduler:
|
||||
def build_connector_meta(
|
||||
self, scheduler_output: SchedulerOutput
|
||||
) -> KVConnectorMetadata:
|
||||
self._update_req_states(scheduler_output)
|
||||
|
||||
# Flush jobs for preempted requests.
|
||||
for req_id in scheduler_output.preempted_req_ids or ():
|
||||
req_status = self._req_status.get(req_id)
|
||||
if req_status is None or not req_status.transfer_jobs:
|
||||
@@ -883,6 +907,20 @@ class OffloadingConnectorScheduler:
|
||||
assert self._jobs[any_jid].is_store
|
||||
self._current_batch_jobs_to_flush.update(req_status.transfer_jobs)
|
||||
|
||||
# Flush jobs that contain re-allocated blocks.
|
||||
if (
|
||||
self._block_id_to_pending_jobs
|
||||
and not self._block_id_to_pending_jobs.keys().isdisjoint(
|
||||
self._current_batch_allocated_block_ids
|
||||
)
|
||||
):
|
||||
self._current_batch_jobs_to_flush.update(
|
||||
jid
|
||||
for bid in self._current_batch_allocated_block_ids
|
||||
if bid in self._block_id_to_pending_jobs
|
||||
for jid in self._block_id_to_pending_jobs[bid]
|
||||
)
|
||||
|
||||
# If all tracked requests are finished, flush all pending jobs
|
||||
# (both store and load) - there might not be a future scheduler
|
||||
# step to trigger their completion.
|
||||
@@ -898,6 +936,7 @@ class OffloadingConnectorScheduler:
|
||||
)
|
||||
self._current_batch_load_jobs = {}
|
||||
self._current_batch_jobs_to_flush = set()
|
||||
self._current_batch_allocated_block_ids = set()
|
||||
return meta
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
@@ -1013,6 +1052,7 @@ class OffloadingConnectorScheduler:
|
||||
# reset_cache cannot be called in the middle of a schedule step
|
||||
assert not self._current_batch_load_jobs
|
||||
assert not self._current_batch_jobs_to_flush
|
||||
assert not self._current_batch_allocated_block_ids
|
||||
|
||||
# Flush all in-flight jobs
|
||||
self._current_batch_jobs_to_flush.update(self._jobs.keys())
|
||||
|
||||
Reference in New Issue
Block a user