Fix KV offload request-finished lifecycle contract (#46284)
Signed-off-by: test test <2260891073@qq.com> Signed-off-by: Rui Yin <2260891073@qq.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -216,17 +216,14 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_no_offload_call_after_on_request_finished(
|
||||
def test_on_request_finished_is_not_deferred_until_store_completion(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""on_request_finished is not issued before a per-request offload
|
||||
call.
|
||||
"""on_request_finished fires when no more stores will be submitted.
|
||||
|
||||
A request can finish while its GPU->primary store is still in flight; the
|
||||
later worker completion then drives complete_store. The scheduler defers
|
||||
on_request_finished until the request is finished AND has no in-flight
|
||||
transfer jobs, so complete_store is observed BEFORE on_request_finished,
|
||||
and it is called exactly once.
|
||||
A request can finish while its GPU->primary store is still in flight. The
|
||||
manager-level hook should not wait for that completion; complete_store may
|
||||
still arrive afterward for already-submitted transfer jobs.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
@@ -263,27 +260,34 @@ def test_no_offload_call_after_on_request_finished(
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
# Finish the request, completing its pending stores. on_request_finished is
|
||||
# deferred until the stores drain, so it lands after the last complete_store.
|
||||
# 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor
|
||||
# GPU blocks.
|
||||
# Finish the request while its stores are still in flight. The hook should
|
||||
# fire immediately even though no complete_store has arrived yet.
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=tuple(range(4 * block_size_factor)),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
req_id = str(runner.req_id)
|
||||
|
||||
assert calls == [("on_request_finished", req_id)], calls
|
||||
|
||||
# Drain the stores afterward. The already-submitted complete_store calls
|
||||
# are allowed to arrive after on_request_finished.
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=True,
|
||||
expected_stored=tuple(range(4 * block_size_factor)),
|
||||
)
|
||||
|
||||
# on_request_finished is issued exactly once.
|
||||
assert calls.count(("on_request_finished", req_id)) == 1, calls
|
||||
|
||||
finished_idx = calls.index(("on_request_finished", req_id))
|
||||
store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)]
|
||||
|
||||
# All of the request's complete_store calls must precede its single
|
||||
# on_request_finished.
|
||||
# The request-level hook no longer waits for already-submitted transfers.
|
||||
assert store_indices, calls
|
||||
assert max(store_indices) < finished_idx, calls
|
||||
assert finished_idx < min(store_indices), calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
@@ -1232,11 +1236,8 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""reset_cache must finalize a finished request whose in-flight stores it
|
||||
discards: call on_request_finished and drop its _req_status entry.
|
||||
|
||||
Otherwise the deferred hook (which waits for the now-discarded jobs to
|
||||
complete) never fires and the entry leaks.
|
||||
"""reset_cache drops a finished request whose in-flight stores it discards
|
||||
without calling on_request_finished twice.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
@@ -1273,14 +1274,15 @@ def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
assert any(job.is_store for job in cs._jobs.values())
|
||||
|
||||
# Finish the request while its store is still in flight. request_finished
|
||||
# takes the defer branch (pending jobs), so on_request_finished is NOT
|
||||
# called yet and the entry stays tracked.
|
||||
# fires the hook eagerly, but the entry stays tracked so later completions
|
||||
# can still call complete_store().
|
||||
req_status.req.status = RequestStatus.FINISHED_STOPPED
|
||||
cs.request_finished(req_status.req)
|
||||
assert finalized == []
|
||||
assert finalized == [req_id]
|
||||
assert req_id in cs._req_status
|
||||
|
||||
# reset_cache discards the in-flight store; it must finalize the request.
|
||||
# reset_cache discards the in-flight store and drops the state without a
|
||||
# duplicate on_request_finished call.
|
||||
cs.reset_cache()
|
||||
assert finalized == [req_id]
|
||||
assert req_id not in cs._req_status
|
||||
|
||||
@@ -235,11 +235,16 @@ class TestTieringOffloadingManager:
|
||||
self.manager.on_schedule_end()
|
||||
list(self.manager.take_events())
|
||||
|
||||
def _start_request(self, req_context: ReqContext = _CTX):
|
||||
if req_context.req_id not in self.manager._req_state:
|
||||
self.manager.on_new_request(req_context)
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 3
|
||||
@@ -262,6 +267,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
@@ -285,6 +291,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
@@ -330,6 +337,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
@@ -366,6 +374,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(5))
|
||||
|
||||
# Store first 3 blocks to primary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:3], _CTX)
|
||||
self.manager.complete_store(blocks[:3], _CTX, success=True)
|
||||
|
||||
@@ -377,6 +386,7 @@ class TestTieringOffloadingManager:
|
||||
# Primary tier has capacity of 5 blocks
|
||||
# First, fill the primary tier
|
||||
blocks = to_keys(range(5))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 5
|
||||
@@ -399,6 +409,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self._simulate_on_schedule_end()
|
||||
@@ -431,6 +442,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
@@ -521,6 +533,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
|
||||
@@ -528,6 +541,172 @@ class TestTieringOffloadingManager:
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert job_metadata.req_context is ctx
|
||||
|
||||
def test_on_request_finished_delays_secondary_until_store_submitted(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||
blocks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
self.primary_tier.on_request_finished = MagicMock(
|
||||
side_effect=lambda req_context: calls.append(
|
||||
("primary_finish", req_context.req_id)
|
||||
)
|
||||
)
|
||||
|
||||
original_submit_store1 = self.secondary_tier1.submit_store
|
||||
original_submit_store2 = self.secondary_tier2.submit_store
|
||||
|
||||
def submit_store1(job_metadata):
|
||||
calls.append(("submit_store_1", job_metadata.req_context.req_id))
|
||||
return original_submit_store1(job_metadata)
|
||||
|
||||
def submit_store2(job_metadata):
|
||||
calls.append(("submit_store_2", job_metadata.req_context.req_id))
|
||||
return original_submit_store2(job_metadata)
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(side_effect=submit_store1)
|
||||
self.secondary_tier2.submit_store = MagicMock(side_effect=submit_store2)
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
side_effect=lambda req_context: calls.append(
|
||||
("secondary_finish_1", req_context.req_id)
|
||||
)
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
side_effect=lambda req_context: calls.append(
|
||||
("secondary_finish_2", req_context.req_id)
|
||||
)
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
assert calls == [("primary_finish", ctx.req_id)]
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
|
||||
assert calls == [
|
||||
("primary_finish", ctx.req_id),
|
||||
("submit_store_1", ctx.req_id),
|
||||
("submit_store_2", ctx.req_id),
|
||||
("secondary_finish_1", ctx.req_id),
|
||||
("secondary_finish_2", ctx.req_id),
|
||||
]
|
||||
|
||||
def test_failed_store_finalizes_finished_request(self, manager_setup):
|
||||
"""Failed primary stores still unblock secondary finalization."""
|
||||
blocks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_failed_store_finalize")
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
)
|
||||
self.secondary_tier2.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=False)
|
||||
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_zero_store_request_finalizes_immediately(self, manager_setup):
|
||||
"""Requests with no pending stores finalize secondary tiers immediately."""
|
||||
ctx = ReqContext(req_id="req_zero_store_finalize")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
|
||||
"""reset_cache abandons pending primary stores and finalizes secondaries."""
|
||||
blocks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_reset_finalize_secondary")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.reset_cache()
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert self.manager._req_state == {}
|
||||
|
||||
def test_reset_cache_clears_pending_primary_stores_for_active_request(
|
||||
self, manager_setup
|
||||
):
|
||||
"""reset_cache drops active pending stores so resumed requests finalize."""
|
||||
initial_blocks = to_keys(range(2))
|
||||
resumed_blocks = to_keys(range(2, 4))
|
||||
ctx = ReqContext(req_id="req_reset_resume")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(initial_blocks, ctx)
|
||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
|
||||
|
||||
self.manager.reset_cache()
|
||||
|
||||
assert ctx.req_id in self.manager._req_state
|
||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 0
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.prepare_store(resumed_blocks, ctx)
|
||||
self.manager.complete_store(resumed_blocks, ctx, success=True)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_on_new_request_lifecycle(self, manager_setup):
|
||||
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
|
||||
and is cleaned up on on_request_finished."""
|
||||
@@ -535,23 +714,25 @@ class TestTieringOffloadingManager:
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.BLOCK_LEVEL
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
# Escalate: tier1 requests REQUEST_LEVEL
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle_2")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.REQUEST_LEVEL
|
||||
assert ctx.req_id in self.manager._request_level_tiers
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
|
||||
self.secondary_tier1
|
||||
}
|
||||
|
||||
# Cleanup
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._request_level_tiers
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
||||
self, manager_setup
|
||||
@@ -559,6 +740,7 @@ class TestTieringOffloadingManager:
|
||||
"""prepare_store cascades hit blocks to request-level tiers only."""
|
||||
# Store some blocks to primary first
|
||||
existing_blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(existing_blocks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(existing_blocks, _CTX, success=True)
|
||||
@@ -566,10 +748,8 @@ class TestTieringOffloadingManager:
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Make tier1 request-level, tier2 stays block-level
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_cascade")
|
||||
@@ -599,14 +779,15 @@ class TestTieringOffloadingManager:
|
||||
# tier2 (block-level) does not get existing blocks here.
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
def test_reset_cache_clears_all_state(self, manager_setup):
|
||||
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
|
||||
"""reset_cache wipes every kind of orchestrator state and resets
|
||||
primary tier; pending submissions are dropped without being sent
|
||||
to the secondary tier."""
|
||||
to the secondary tier. Active request state is retained."""
|
||||
# Cascade — populates primary blocks and leaves cascade jobs
|
||||
# in _transfer_jobs (the synchronous example tier has already
|
||||
# queued completions); reset_cache's drain loop will pick them up.
|
||||
blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
assert self.manager._transfer_jobs
|
||||
@@ -619,13 +800,14 @@ class TestTieringOffloadingManager:
|
||||
assert self.manager._pending_load_submissions
|
||||
|
||||
# Request-level tier registration.
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
self.manager.on_new_request(ReqContext(req_id="rl"))
|
||||
assert self.manager._request_level_tiers
|
||||
rl_ctx = ReqContext(req_id="rl")
|
||||
self.manager.on_new_request(rl_ctx)
|
||||
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
|
||||
self.secondary_tier1
|
||||
}
|
||||
|
||||
# Mark this step as already polled (reset_cache must clear it).
|
||||
self.manager._processed_jobs_this_step = True
|
||||
@@ -640,7 +822,7 @@ class TestTieringOffloadingManager:
|
||||
# Orchestrator state cleared.
|
||||
assert self.manager._transfer_jobs == {}
|
||||
assert self.manager._pending_load_submissions == {}
|
||||
assert self.manager._request_level_tiers == {}
|
||||
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
|
||||
# Primary tier reset to a fresh state.
|
||||
@@ -668,6 +850,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Drive a cascade so a job lands in _transfer_jobs.
|
||||
blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
assert self.manager._transfer_jobs
|
||||
@@ -696,6 +879,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Should work like a regular OffloadingManager
|
||||
manager.on_new_request(_CTX)
|
||||
result = manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
Reference in New Issue
Block a user