[Fix][KV offload] Defer on_request_finished until in-flight transfers drain (#45823)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
Ronen Schaffer
2026-06-18 07:05:46 +03:00
committed by GitHub
parent 4403af8fb5
commit f428718ffe
4 changed files with 174 additions and 7 deletions
@@ -243,6 +243,77 @@ def test_request_preemption(request_runner, async_scheduling: bool):
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_no_offload_call_after_on_request_finished(
request_runner, async_scheduling: bool
):
"""on_request_finished is not issued before a per-request offload
call.
A request can finish while its GPU->primary store is still in flight; the
later worker completion then drives complete_store. The scheduler defers
on_request_finished until the request is finished AND has no in-flight
transfer jobs, so complete_store is observed BEFORE on_request_finished,
and it is called exactly once.
"""
block_size = 4
block_size_factor = 3
offloaded_block_size = block_size * block_size_factor
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
block_size_factor=block_size_factor,
)
# Record the order of per-request connector calls on the (mocked) manager.
# The external list survives manager.reset_mock() between run() calls.
calls: list[tuple[str, str]] = []
runner.manager.on_request_finished.side_effect = lambda req_context: calls.append(
("on_request_finished", req_context.req_id)
)
runner.manager.complete_store.side_effect = (
lambda keys, req_context, *args, **kwargs: calls.append(
("complete_store", req_context.req_id)
)
)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
# Decode a couple of blocks, keeping every transfer in flight
# (complete_transfers=False) so no store completes while the request runs.
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
runner.run(decoded_tokens=[0], complete_transfers=False)
runner.run(
decoded_tokens=[0] * (2 * offloaded_block_size),
complete_transfers=False,
)
# Finish the request, completing its pending stores. on_request_finished is
# deferred until the stores drain, so it lands after the last complete_store.
# 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor
# GPU blocks.
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=tuple(range(4 * block_size_factor)),
)
req_id = str(runner.req_id)
# on_request_finished is issued exactly once.
assert calls.count(("on_request_finished", req_id)) == 1, calls
finished_idx = calls.index(("on_request_finished", req_id))
store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)]
# All of the request's complete_store calls must precede its single
# on_request_finished.
assert store_indices, calls
assert max(store_indices) < finished_idx, calls
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool):
block_size = 4
@@ -1149,6 +1220,64 @@ def test_reset_cache(request_runner, async_scheduling: bool):
assert group_state.next_stored_block_idx == 0
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_reset_cache_finalizes_finished_request_with_pending_store(
request_runner, async_scheduling: bool
):
"""reset_cache must finalize a finished request whose in-flight stores it
discards: call on_request_finished and drop its _req_status entry.
Otherwise the deferred hook (which waits for the now-discarded jobs to
complete) never fires and the entry leaks.
"""
block_size = 4
block_size_factor = 3
offloaded_block_size = block_size * block_size_factor
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
block_size_factor=block_size_factor,
)
finalized: list[str] = []
runner.manager.on_request_finished.side_effect = (
lambda req_context: finalized.append(req_context.req_id)
)
runner.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
# Decode a couple of blocks and keep every transfer in flight, so the
# request has pending store jobs.
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
runner.run(decoded_tokens=[0], complete_transfers=False)
runner.run(
decoded_tokens=[0] * (2 * offloaded_block_size),
complete_transfers=False,
)
cs = runner.connector_scheduler
req_id = str(runner.req_id)
req_status = cs._req_status[req_id]
assert req_status.transfer_jobs, "expected an in-flight store before finish"
assert any(job.is_store for job in cs._jobs.values())
# Finish the request while its store is still in flight. request_finished
# takes the defer branch (pending jobs), so on_request_finished is NOT
# called yet and the entry stays tracked.
req_status.req.status = RequestStatus.FINISHED_STOPPED
cs.request_finished(req_status.req)
assert finalized == []
assert req_id in cs._req_status
# reset_cache discards the in-flight store; it must finalize the request.
cs.reset_cache()
assert finalized == [req_id]
assert req_id not in cs._req_status
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
"""SWA blocks unreachable by the load path are skipped during store.
@@ -1115,6 +1115,11 @@ class OffloadingConnectorScheduler:
del self._jobs[job_id]
req_status.transfer_jobs.remove(job_id)
if not req_status.transfer_jobs and req_status.req.is_finished():
# Deferred from request_finished: the request's last in-flight
# job is now done, so fire the finalize hook here, after the
# final complete_store/complete_load above (and any submit_store
# the complete_store cascade issued).
self.manager.on_request_finished(req_status.req_context)
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
@@ -1148,18 +1153,23 @@ class OffloadingConnectorScheduler:
# which may have been deferred due to async scheduling
req_status = self._req_status.get(request.request_id)
req_context = (
req_status.req_context if req_status else _create_req_context(request)
)
self.manager.on_request_finished(req_context)
if req_status is None:
# Untracked request (offloading never started): no in-flight jobs,
# nothing was deferred, so finalize immediately.
self.manager.on_request_finished(_create_req_context(request))
return False, None
if not req_status.transfer_jobs:
# No in-flight jobs: all per-request calls are done, finalize now.
self.manager.on_request_finished(req_status.req_context)
del self._req_status[request.request_id]
return False, None
# Pending stores will outlive the request's block ownership.
# Register them so future block reuse triggers a flush.
# In-flight jobs remain, so defer on_request_finished to
# update_connector_output, which fires it once the last job completes
# (after the final complete_store and any cascade submit_store it
# issues). These pending stores outlive the request's block ownership;
# register them so future reuse of those blocks triggers a flush.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.non_sliding_window_block_ids or ():
@@ -1198,6 +1208,16 @@ class OffloadingConnectorScheduler:
# Flush all in-flight jobs
self._current_batch_jobs_to_flush.update(self._jobs.keys())
# A finished request may still be tracked here with in-flight jobs that
# this reset discards, so its deferred on_request_finished() would never
# fire (completions are skipped as stale) and its _req_status entry would
# leak. Finalize such requests now, before resetting the manager.
# list() snapshots because we delete while iterating.
for req_id, status in list(self._req_status.items()):
if status.req.is_finished():
self.manager.on_request_finished(status.req_context)
del self._req_status[req_id]
# Reset offloading manager cache
self.manager.reset_cache()
+11
View File
@@ -266,6 +266,17 @@ class OffloadingManager(ABC):
"""
Called when a request has finished.
By the time this is called, all per-request offload calls for this
request (prepare_store/complete_store, prepare_load/complete_load,
touch, lookup) have already been issued, and none will follow. The
scheduler defers this call until the request is finished and has no
in-flight transfer jobs.
Note this signals only that no further calls will be made; it does NOT
imply the data has been persisted. Asynchronous transfers already
submitted for this request (e.g. CPU->secondary cascades) may still be
in flight. This is the right place to release per-request bookkeeping.
Args:
req_context: per-request context.
"""
+7
View File
@@ -188,6 +188,13 @@ class SecondaryTierManager(ABC):
"""
Called when a request has finished.
By the time this is called, all per-request calls for this request
(submit_store, submit_load, touch) have already been issued, and none
will follow. Note this does NOT imply the tier's transfers have
completed: jobs already submitted may still be in flight and will
report via get_finished_jobs(). This is the right place to release
per-request bookkeeping.
Args:
req_context: per-request context.
"""