Fix KV offload request-finished lifecycle contract (#46284)

Signed-off-by: test test <2260891073@qq.com>
Signed-off-by: Rui Yin <2260891073@qq.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Rui Yin
2026-06-24 15:42:40 +03:00
committed by GitHub
co-authored by Or Ozeri
parent d4448b511d
commit cf9fd6457e
5 changed files with 334 additions and 112 deletions
@@ -216,17 +216,14 @@ def test_request_preemption(request_runner, async_scheduling: bool):
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_no_offload_call_after_on_request_finished(
def test_on_request_finished_is_not_deferred_until_store_completion(
request_runner, async_scheduling: bool
):
"""on_request_finished is not issued before a per-request offload
call.
"""on_request_finished fires when no more stores will be submitted.
A request can finish while its GPU->primary store is still in flight; the
later worker completion then drives complete_store. The scheduler defers
on_request_finished until the request is finished AND has no in-flight
transfer jobs, so complete_store is observed BEFORE on_request_finished,
and it is called exactly once.
A request can finish while its GPU->primary store is still in flight. The
manager-level hook should not wait for that completion; complete_store may
still arrive afterward for already-submitted transfer jobs.
"""
block_size = 4
block_size_factor = 3
@@ -263,27 +260,34 @@ def test_no_offload_call_after_on_request_finished(
complete_transfers=False,
)
# Finish the request, completing its pending stores. on_request_finished is
# deferred until the stores drain, so it lands after the last complete_store.
# 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor
# GPU blocks.
# Finish the request while its stores are still in flight. The hook should
# fire immediately even though no complete_store has arrived yet.
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=tuple(range(4 * block_size_factor)),
complete_transfers=False,
)
req_id = str(runner.req_id)
assert calls == [("on_request_finished", req_id)], calls
# Drain the stores afterward. The already-submitted complete_store calls
# are allowed to arrive after on_request_finished.
runner.run(
decoded_tokens=[],
complete_transfers=True,
expected_stored=tuple(range(4 * block_size_factor)),
)
# on_request_finished is issued exactly once.
assert calls.count(("on_request_finished", req_id)) == 1, calls
finished_idx = calls.index(("on_request_finished", req_id))
store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)]
# All of the request's complete_store calls must precede its single
# on_request_finished.
# The request-level hook no longer waits for already-submitted transfers.
assert store_indices, calls
assert max(store_indices) < finished_idx, calls
assert finished_idx < min(store_indices), calls
@pytest.mark.parametrize("async_scheduling", [True, False])
@@ -1232,11 +1236,8 @@ def test_reset_cache(request_runner, async_scheduling: bool):
def test_reset_cache_finalizes_finished_request_with_pending_store(
request_runner, async_scheduling: bool
):
"""reset_cache must finalize a finished request whose in-flight stores it
discards: call on_request_finished and drop its _req_status entry.
Otherwise the deferred hook (which waits for the now-discarded jobs to
complete) never fires and the entry leaks.
"""reset_cache drops a finished request whose in-flight stores it discards
without calling on_request_finished twice.
"""
block_size = 4
block_size_factor = 3
@@ -1273,14 +1274,15 @@ def test_reset_cache_finalizes_finished_request_with_pending_store(
assert any(job.is_store for job in cs._jobs.values())
# Finish the request while its store is still in flight. request_finished
# takes the defer branch (pending jobs), so on_request_finished is NOT
# called yet and the entry stays tracked.
# fires the hook eagerly, but the entry stays tracked so later completions
# can still call complete_store().
req_status.req.status = RequestStatus.FINISHED_STOPPED
cs.request_finished(req_status.req)
assert finalized == []
assert finalized == [req_id]
assert req_id in cs._req_status
# reset_cache discards the in-flight store; it must finalize the request.
# reset_cache discards the in-flight store and drops the state without a
# duplicate on_request_finished call.
cs.reset_cache()
assert finalized == [req_id]
assert req_id not in cs._req_status
@@ -235,11 +235,16 @@ class TestTieringOffloadingManager:
self.manager.on_schedule_end()
list(self.manager.take_events())
def _start_request(self, req_context: ReqContext = _CTX):
if req_context.req_id not in self.manager._req_state:
self.manager.on_new_request(req_context)
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))
# Prepare store
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 3
@@ -262,6 +267,7 @@ class TestTieringOffloadingManager:
)
# Store to primary
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
@@ -285,6 +291,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(3))
# Store to primary
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
self.manager.complete_store(blocks, _CTX, success=True)
@@ -330,6 +337,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(3))
# Store blocks
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
@@ -366,6 +374,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(5))
# Store first 3 blocks to primary
self._start_request()
self.manager.prepare_store(blocks[:3], _CTX)
self.manager.complete_store(blocks[:3], _CTX, success=True)
@@ -377,6 +386,7 @@ class TestTieringOffloadingManager:
# Primary tier has capacity of 5 blocks
# First, fill the primary tier
blocks = to_keys(range(5))
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 5
@@ -399,6 +409,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(3))
# Store blocks
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
self._simulate_on_schedule_end()
@@ -431,6 +442,7 @@ class TestTieringOffloadingManager:
)
# Prepare store
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
@@ -521,6 +533,7 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.complete_store(blocks, ctx, success=True)
@@ -528,6 +541,172 @@ class TestTieringOffloadingManager:
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx
def test_on_request_finished_delays_secondary_until_store_submitted(
self, manager_setup
):
"""Manager hook is eager; secondary hooks wait for cascade submission."""
blocks = to_keys(range(2))
ctx = ReqContext(req_id="req_delayed_secondary")
calls: list[tuple[str, str]] = []
self.primary_tier.on_request_finished = MagicMock(
side_effect=lambda req_context: calls.append(
("primary_finish", req_context.req_id)
)
)
original_submit_store1 = self.secondary_tier1.submit_store
original_submit_store2 = self.secondary_tier2.submit_store
def submit_store1(job_metadata):
calls.append(("submit_store_1", job_metadata.req_context.req_id))
return original_submit_store1(job_metadata)
def submit_store2(job_metadata):
calls.append(("submit_store_2", job_metadata.req_context.req_id))
return original_submit_store2(job_metadata)
self.secondary_tier1.submit_store = MagicMock(side_effect=submit_store1)
self.secondary_tier2.submit_store = MagicMock(side_effect=submit_store2)
self.secondary_tier1.on_request_finished = MagicMock(
side_effect=lambda req_context: calls.append(
("secondary_finish_1", req_context.req_id)
)
)
self.secondary_tier2.on_request_finished = MagicMock(
side_effect=lambda req_context: calls.append(
("secondary_finish_2", req_context.req_id)
)
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.on_request_finished(ctx)
assert calls == [("primary_finish", ctx.req_id)]
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=True)
assert calls == [
("primary_finish", ctx.req_id),
("submit_store_1", ctx.req_id),
("submit_store_2", ctx.req_id),
("secondary_finish_1", ctx.req_id),
("secondary_finish_2", ctx.req_id),
]
def test_failed_store_finalizes_finished_request(self, manager_setup):
"""Failed primary stores still unblock secondary finalization."""
blocks = to_keys(range(2))
ctx = ReqContext(req_id="req_failed_store_finalize")
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=False)
self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called()
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert ctx.req_id not in self.manager._req_state
def test_zero_store_request_finalizes_immediately(self, manager_setup):
"""Requests with no pending stores finalize secondary tiers immediately."""
ctx = ReqContext(req_id="req_zero_store_finalize")
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert ctx.req_id not in self.manager._req_state
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
"""reset_cache abandons pending primary stores and finalizes secondaries."""
blocks = to_keys(range(2))
ctx = ReqContext(req_id="req_reset_finalize_secondary")
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.reset_cache()
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert self.manager._req_state == {}
def test_reset_cache_clears_pending_primary_stores_for_active_request(
self, manager_setup
):
"""reset_cache drops active pending stores so resumed requests finalize."""
initial_blocks = to_keys(range(2))
resumed_blocks = to_keys(range(2, 4))
ctx = ReqContext(req_id="req_reset_resume")
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.prepare_store(initial_blocks, ctx)
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
self.manager.reset_cache()
assert ctx.req_id in self.manager._req_state
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 0
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.prepare_store(resumed_blocks, ctx)
self.manager.complete_store(resumed_blocks, ctx, success=True)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert ctx.req_id not in self.manager._req_state
def test_on_new_request_lifecycle(self, manager_setup):
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
and is cleaned up on on_request_finished."""
@@ -535,23 +714,25 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_policy_lifecycle")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.BLOCK_LEVEL
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._req_state
# Escalate: tier1 requests REQUEST_LEVEL
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
ctx = ReqContext(req_id="req_policy_lifecycle_2")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.REQUEST_LEVEL
assert ctx.req_id in self.manager._request_level_tiers
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
self.secondary_tier1
}
# Cleanup
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._request_level_tiers
assert ctx.req_id not in self.manager._req_state
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
self, manager_setup
@@ -559,6 +740,7 @@ class TestTieringOffloadingManager:
"""prepare_store cascades hit blocks to request-level tiers only."""
# Store some blocks to primary first
existing_blocks = to_keys(range(3))
self._start_request()
result = self.manager.prepare_store(existing_blocks, _CTX)
assert result is not None
self.manager.complete_store(existing_blocks, _CTX, success=True)
@@ -566,10 +748,8 @@ class TestTieringOffloadingManager:
self._simulate_on_schedule_end()
# Make tier1 request-level, tier2 stays block-level
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
ctx = ReqContext(req_id="req_cascade")
@@ -599,14 +779,15 @@ class TestTieringOffloadingManager:
# tier2 (block-level) does not get existing blocks here.
self.secondary_tier2.submit_store.assert_not_called()
def test_reset_cache_clears_all_state(self, manager_setup):
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
"""reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent
to the secondary tier."""
to the secondary tier. Active request state is retained."""
# Cascade — populates primary blocks and leaves cascade jobs
# in _transfer_jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up.
blocks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
@@ -619,13 +800,14 @@ class TestTieringOffloadingManager:
assert self.manager._pending_load_submissions
# Request-level tier registration.
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
self.manager.on_new_request(ReqContext(req_id="rl"))
assert self.manager._request_level_tiers
rl_ctx = ReqContext(req_id="rl")
self.manager.on_new_request(rl_ctx)
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
self.secondary_tier1
}
# Mark this step as already polled (reset_cache must clear it).
self.manager._processed_jobs_this_step = True
@@ -640,7 +822,7 @@ class TestTieringOffloadingManager:
# Orchestrator state cleared.
assert self.manager._transfer_jobs == {}
assert self.manager._pending_load_submissions == {}
assert self.manager._request_level_tiers == {}
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
assert self.manager._processed_jobs_this_step is False
# Primary tier reset to a fresh state.
@@ -668,6 +850,7 @@ class TestTieringOffloadingManager:
# Drive a cascade so a job lands in _transfer_jobs.
blocks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
@@ -696,6 +879,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
blocks = to_keys(range(3))
# Should work like a regular OffloadingManager
manager.on_new_request(_CTX)
result = manager.prepare_store(blocks, _CTX)
assert result is not None
manager.complete_store(blocks, _CTX, success=True)