[Fix][KV offload] Defer on_request_finished until in-flight transfers drain (#45823)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
This commit is contained in:
@@ -243,6 +243,77 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
assert runner.connector_scheduler._block_id_to_pending_jobs == {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_no_offload_call_after_on_request_finished(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""on_request_finished is not issued before a per-request offload
|
||||
call.
|
||||
|
||||
A request can finish while its GPU->primary store is still in flight; the
|
||||
later worker completion then drives complete_store. The scheduler defers
|
||||
on_request_finished until the request is finished AND has no in-flight
|
||||
transfer jobs, so complete_store is observed BEFORE on_request_finished,
|
||||
and it is called exactly once.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
)
|
||||
|
||||
# Record the order of per-request connector calls on the (mocked) manager.
|
||||
# The external list survives manager.reset_mock() between run() calls.
|
||||
calls: list[tuple[str, str]] = []
|
||||
runner.manager.on_request_finished.side_effect = lambda req_context: calls.append(
|
||||
("on_request_finished", req_context.req_id)
|
||||
)
|
||||
runner.manager.complete_store.side_effect = (
|
||||
lambda keys, req_context, *args, **kwargs: calls.append(
|
||||
("complete_store", req_context.req_id)
|
||||
)
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
|
||||
# Decode a couple of blocks, keeping every transfer in flight
|
||||
# (complete_transfers=False) so no store completes while the request runs.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.run(decoded_tokens=[0], complete_transfers=False)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (2 * offloaded_block_size),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
# Finish the request, completing its pending stores. on_request_finished is
|
||||
# deferred until the stores drain, so it lands after the last complete_store.
|
||||
# 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor
|
||||
# GPU blocks.
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=tuple(range(4 * block_size_factor)),
|
||||
)
|
||||
|
||||
req_id = str(runner.req_id)
|
||||
|
||||
# on_request_finished is issued exactly once.
|
||||
assert calls.count(("on_request_finished", req_id)) == 1, calls
|
||||
|
||||
finished_idx = calls.index(("on_request_finished", req_id))
|
||||
store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)]
|
||||
|
||||
# All of the request's complete_store calls must precede its single
|
||||
# on_request_finished.
|
||||
assert store_indices, calls
|
||||
assert max(store_indices) < finished_idx, calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_concurrent_lookups_of_the_same_prefix(request_runner, async_scheduling: bool):
|
||||
block_size = 4
|
||||
@@ -1149,6 +1220,64 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
assert group_state.next_stored_block_idx == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""reset_cache must finalize a finished request whose in-flight stores it
|
||||
discards: call on_request_finished and drop its _req_status entry.
|
||||
|
||||
Otherwise the deferred hook (which waits for the now-discarded jobs to
|
||||
complete) never fires and the entry leaks.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = block_size * block_size_factor
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
block_size_factor=block_size_factor,
|
||||
)
|
||||
|
||||
finalized: list[str] = []
|
||||
runner.manager.on_request_finished.side_effect = (
|
||||
lambda req_context: finalized.append(req_context.req_id)
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
|
||||
# Decode a couple of blocks and keep every transfer in flight, so the
|
||||
# request has pending store jobs.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.run(decoded_tokens=[0], complete_transfers=False)
|
||||
runner.run(
|
||||
decoded_tokens=[0] * (2 * offloaded_block_size),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
cs = runner.connector_scheduler
|
||||
req_id = str(runner.req_id)
|
||||
req_status = cs._req_status[req_id]
|
||||
assert req_status.transfer_jobs, "expected an in-flight store before finish"
|
||||
assert any(job.is_store for job in cs._jobs.values())
|
||||
|
||||
# Finish the request while its store is still in flight. request_finished
|
||||
# takes the defer branch (pending jobs), so on_request_finished is NOT
|
||||
# called yet and the entry stays tracked.
|
||||
req_status.req.status = RequestStatus.FINISHED_STOPPED
|
||||
cs.request_finished(req_status.req)
|
||||
assert finalized == []
|
||||
assert req_id in cs._req_status
|
||||
|
||||
# reset_cache discards the in-flight store; it must finalize the request.
|
||||
cs.reset_cache()
|
||||
assert finalized == [req_id]
|
||||
assert req_id not in cs._req_status
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_swa_alignment_skip(request_runner, async_scheduling: bool):
|
||||
"""SWA blocks unreachable by the load path are skipped during store.
|
||||
|
||||
@@ -1115,6 +1115,11 @@ class OffloadingConnectorScheduler:
|
||||
del self._jobs[job_id]
|
||||
req_status.transfer_jobs.remove(job_id)
|
||||
if not req_status.transfer_jobs and req_status.req.is_finished():
|
||||
# Deferred from request_finished: the request's last in-flight
|
||||
# job is now done, so fire the finalize hook here, after the
|
||||
# final complete_store/complete_load above (and any submit_store
|
||||
# the complete_store cascade issued).
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
@@ -1148,18 +1153,23 @@ class OffloadingConnectorScheduler:
|
||||
# which may have been deferred due to async scheduling
|
||||
req_status = self._req_status.get(request.request_id)
|
||||
|
||||
req_context = (
|
||||
req_status.req_context if req_status else _create_req_context(request)
|
||||
)
|
||||
self.manager.on_request_finished(req_context)
|
||||
|
||||
if req_status is None:
|
||||
# Untracked request (offloading never started): no in-flight jobs,
|
||||
# nothing was deferred, so finalize immediately.
|
||||
self.manager.on_request_finished(_create_req_context(request))
|
||||
return False, None
|
||||
|
||||
if not req_status.transfer_jobs:
|
||||
# No in-flight jobs: all per-request calls are done, finalize now.
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
del self._req_status[request.request_id]
|
||||
return False, None
|
||||
# Pending stores will outlive the request's block ownership.
|
||||
# Register them so future block reuse triggers a flush.
|
||||
|
||||
# In-flight jobs remain, so defer on_request_finished to
|
||||
# update_connector_output, which fires it once the last job completes
|
||||
# (after the final complete_store and any cascade submit_store it
|
||||
# issues). These pending stores outlive the request's block ownership;
|
||||
# register them so future reuse of those blocks triggers a flush.
|
||||
for job_id in req_status.transfer_jobs:
|
||||
job_status = self._jobs[job_id]
|
||||
for bid in job_status.non_sliding_window_block_ids or ():
|
||||
@@ -1198,6 +1208,16 @@ class OffloadingConnectorScheduler:
|
||||
# Flush all in-flight jobs
|
||||
self._current_batch_jobs_to_flush.update(self._jobs.keys())
|
||||
|
||||
# A finished request may still be tracked here with in-flight jobs that
|
||||
# this reset discards, so its deferred on_request_finished() would never
|
||||
# fire (completions are skipped as stale) and its _req_status entry would
|
||||
# leak. Finalize such requests now, before resetting the manager.
|
||||
# list() snapshots because we delete while iterating.
|
||||
for req_id, status in list(self._req_status.items()):
|
||||
if status.req.is_finished():
|
||||
self.manager.on_request_finished(status.req_context)
|
||||
del self._req_status[req_id]
|
||||
|
||||
# Reset offloading manager cache
|
||||
self.manager.reset_cache()
|
||||
|
||||
|
||||
@@ -266,6 +266,17 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
Called when a request has finished.
|
||||
|
||||
By the time this is called, all per-request offload calls for this
|
||||
request (prepare_store/complete_store, prepare_load/complete_load,
|
||||
touch, lookup) have already been issued, and none will follow. The
|
||||
scheduler defers this call until the request is finished and has no
|
||||
in-flight transfer jobs.
|
||||
|
||||
Note this signals only that no further calls will be made; it does NOT
|
||||
imply the data has been persisted. Asynchronous transfers already
|
||||
submitted for this request (e.g. CPU->secondary cascades) may still be
|
||||
in flight. This is the right place to release per-request bookkeeping.
|
||||
|
||||
Args:
|
||||
req_context: per-request context.
|
||||
"""
|
||||
|
||||
@@ -188,6 +188,13 @@ class SecondaryTierManager(ABC):
|
||||
"""
|
||||
Called when a request has finished.
|
||||
|
||||
By the time this is called, all per-request calls for this request
|
||||
(submit_store, submit_load, touch) have already been issued, and none
|
||||
will follow. Note this does NOT imply the tier's transfers have
|
||||
completed: jobs already submitted may still be in flight and will
|
||||
report via get_finished_jobs(). This is the right place to release
|
||||
per-request bookkeeping.
|
||||
|
||||
Args:
|
||||
req_context: per-request context.
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user