Fix KV offload request-finished lifecycle contract (#46284)
Signed-off-by: test test <2260891073@qq.com> Signed-off-by: Rui Yin <2260891073@qq.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -216,17 +216,14 @@ def test_request_preemption(request_runner, async_scheduling: bool):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_no_offload_call_after_on_request_finished(
|
||||
def test_on_request_finished_is_not_deferred_until_store_completion(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""on_request_finished is not issued before a per-request offload
|
||||
call.
|
||||
"""on_request_finished fires when no more stores will be submitted.
|
||||
|
||||
A request can finish while its GPU->primary store is still in flight; the
|
||||
later worker completion then drives complete_store. The scheduler defers
|
||||
on_request_finished until the request is finished AND has no in-flight
|
||||
transfer jobs, so complete_store is observed BEFORE on_request_finished,
|
||||
and it is called exactly once.
|
||||
A request can finish while its GPU->primary store is still in flight. The
|
||||
manager-level hook should not wait for that completion; complete_store may
|
||||
still arrive afterward for already-submitted transfer jobs.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
@@ -263,27 +260,34 @@ def test_no_offload_call_after_on_request_finished(
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
# Finish the request, completing its pending stores. on_request_finished is
|
||||
# deferred until the stores drain, so it lands after the last complete_store.
|
||||
# 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor
|
||||
# GPU blocks.
|
||||
# Finish the request while its stores are still in flight. The hook should
|
||||
# fire immediately even though no complete_store has arrived yet.
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=tuple(range(4 * block_size_factor)),
|
||||
complete_transfers=False,
|
||||
)
|
||||
|
||||
req_id = str(runner.req_id)
|
||||
|
||||
assert calls == [("on_request_finished", req_id)], calls
|
||||
|
||||
# Drain the stores afterward. The already-submitted complete_store calls
|
||||
# are allowed to arrive after on_request_finished.
|
||||
runner.run(
|
||||
decoded_tokens=[],
|
||||
complete_transfers=True,
|
||||
expected_stored=tuple(range(4 * block_size_factor)),
|
||||
)
|
||||
|
||||
# on_request_finished is issued exactly once.
|
||||
assert calls.count(("on_request_finished", req_id)) == 1, calls
|
||||
|
||||
finished_idx = calls.index(("on_request_finished", req_id))
|
||||
store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)]
|
||||
|
||||
# All of the request's complete_store calls must precede its single
|
||||
# on_request_finished.
|
||||
# The request-level hook no longer waits for already-submitted transfers.
|
||||
assert store_indices, calls
|
||||
assert max(store_indices) < finished_idx, calls
|
||||
assert finished_idx < min(store_indices), calls
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
@@ -1232,11 +1236,8 @@ def test_reset_cache(request_runner, async_scheduling: bool):
|
||||
def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
request_runner, async_scheduling: bool
|
||||
):
|
||||
"""reset_cache must finalize a finished request whose in-flight stores it
|
||||
discards: call on_request_finished and drop its _req_status entry.
|
||||
|
||||
Otherwise the deferred hook (which waits for the now-discarded jobs to
|
||||
complete) never fires and the entry leaks.
|
||||
"""reset_cache drops a finished request whose in-flight stores it discards
|
||||
without calling on_request_finished twice.
|
||||
"""
|
||||
block_size = 4
|
||||
block_size_factor = 3
|
||||
@@ -1273,14 +1274,15 @@ def test_reset_cache_finalizes_finished_request_with_pending_store(
|
||||
assert any(job.is_store for job in cs._jobs.values())
|
||||
|
||||
# Finish the request while its store is still in flight. request_finished
|
||||
# takes the defer branch (pending jobs), so on_request_finished is NOT
|
||||
# called yet and the entry stays tracked.
|
||||
# fires the hook eagerly, but the entry stays tracked so later completions
|
||||
# can still call complete_store().
|
||||
req_status.req.status = RequestStatus.FINISHED_STOPPED
|
||||
cs.request_finished(req_status.req)
|
||||
assert finalized == []
|
||||
assert finalized == [req_id]
|
||||
assert req_id in cs._req_status
|
||||
|
||||
# reset_cache discards the in-flight store; it must finalize the request.
|
||||
# reset_cache discards the in-flight store and drops the state without a
|
||||
# duplicate on_request_finished call.
|
||||
cs.reset_cache()
|
||||
assert finalized == [req_id]
|
||||
assert req_id not in cs._req_status
|
||||
|
||||
@@ -235,11 +235,16 @@ class TestTieringOffloadingManager:
|
||||
self.manager.on_schedule_end()
|
||||
list(self.manager.take_events())
|
||||
|
||||
def _start_request(self, req_context: ReqContext = _CTX):
|
||||
if req_context.req_id not in self.manager._req_state:
|
||||
self.manager.on_new_request(req_context)
|
||||
|
||||
def test_basic_store_to_primary(self, manager_setup):
|
||||
"""Test basic store operation to primary tier."""
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 3
|
||||
@@ -262,6 +267,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
@@ -285,6 +291,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store to primary
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
@@ -330,6 +337,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
@@ -366,6 +374,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(5))
|
||||
|
||||
# Store first 3 blocks to primary
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks[:3], _CTX)
|
||||
self.manager.complete_store(blocks[:3], _CTX, success=True)
|
||||
|
||||
@@ -377,6 +386,7 @@ class TestTieringOffloadingManager:
|
||||
# Primary tier has capacity of 5 blocks
|
||||
# First, fill the primary tier
|
||||
blocks = to_keys(range(5))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
assert len(result.keys_to_store) == 5
|
||||
@@ -399,6 +409,7 @@ class TestTieringOffloadingManager:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Store blocks
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
self._simulate_on_schedule_end()
|
||||
@@ -431,6 +442,7 @@ class TestTieringOffloadingManager:
|
||||
)
|
||||
|
||||
# Prepare store
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
|
||||
@@ -521,6 +533,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
|
||||
@@ -528,6 +541,172 @@ class TestTieringOffloadingManager:
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert job_metadata.req_context is ctx
|
||||
|
||||
def test_on_request_finished_delays_secondary_until_store_submitted(
|
||||
self, manager_setup
|
||||
):
|
||||
"""Manager hook is eager; secondary hooks wait for cascade submission."""
|
||||
blocks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_delayed_secondary")
|
||||
calls: list[tuple[str, str]] = []
|
||||
|
||||
self.primary_tier.on_request_finished = MagicMock(
|
||||
side_effect=lambda req_context: calls.append(
|
||||
("primary_finish", req_context.req_id)
|
||||
)
|
||||
)
|
||||
|
||||
original_submit_store1 = self.secondary_tier1.submit_store
|
||||
original_submit_store2 = self.secondary_tier2.submit_store
|
||||
|
||||
def submit_store1(job_metadata):
|
||||
calls.append(("submit_store_1", job_metadata.req_context.req_id))
|
||||
return original_submit_store1(job_metadata)
|
||||
|
||||
def submit_store2(job_metadata):
|
||||
calls.append(("submit_store_2", job_metadata.req_context.req_id))
|
||||
return original_submit_store2(job_metadata)
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(side_effect=submit_store1)
|
||||
self.secondary_tier2.submit_store = MagicMock(side_effect=submit_store2)
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
side_effect=lambda req_context: calls.append(
|
||||
("secondary_finish_1", req_context.req_id)
|
||||
)
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
side_effect=lambda req_context: calls.append(
|
||||
("secondary_finish_2", req_context.req_id)
|
||||
)
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
assert calls == [("primary_finish", ctx.req_id)]
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=True)
|
||||
|
||||
assert calls == [
|
||||
("primary_finish", ctx.req_id),
|
||||
("submit_store_1", ctx.req_id),
|
||||
("submit_store_2", ctx.req_id),
|
||||
("secondary_finish_1", ctx.req_id),
|
||||
("secondary_finish_2", ctx.req_id),
|
||||
]
|
||||
|
||||
def test_failed_store_finalizes_finished_request(self, manager_setup):
|
||||
"""Failed primary stores still unblock secondary finalization."""
|
||||
blocks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_failed_store_finalize")
|
||||
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
)
|
||||
self.secondary_tier2.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.complete_store(blocks, ctx, success=False)
|
||||
|
||||
self.secondary_tier1.submit_store.assert_not_called()
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_zero_store_request_finalizes_immediately(self, manager_setup):
|
||||
"""Requests with no pending stores finalize secondary tiers immediately."""
|
||||
ctx = ReqContext(req_id="req_zero_store_finalize")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
|
||||
"""reset_cache abandons pending primary stores and finalizes secondaries."""
|
||||
blocks = to_keys(range(2))
|
||||
ctx = ReqContext(req_id="req_reset_finalize_secondary")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(blocks, ctx)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.reset_cache()
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert self.manager._req_state == {}
|
||||
|
||||
def test_reset_cache_clears_pending_primary_stores_for_active_request(
|
||||
self, manager_setup
|
||||
):
|
||||
"""reset_cache drops active pending stores so resumed requests finalize."""
|
||||
initial_blocks = to_keys(range(2))
|
||||
resumed_blocks = to_keys(range(2, 4))
|
||||
ctx = ReqContext(req_id="req_reset_resume")
|
||||
|
||||
self.secondary_tier1.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier1.on_request_finished
|
||||
)
|
||||
self.secondary_tier2.on_request_finished = MagicMock(
|
||||
wraps=self.secondary_tier2.on_request_finished
|
||||
)
|
||||
|
||||
self._start_request(ctx)
|
||||
self.manager.prepare_store(initial_blocks, ctx)
|
||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
|
||||
|
||||
self.manager.reset_cache()
|
||||
|
||||
assert ctx.req_id in self.manager._req_state
|
||||
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 0
|
||||
self.secondary_tier1.on_request_finished.assert_not_called()
|
||||
self.secondary_tier2.on_request_finished.assert_not_called()
|
||||
|
||||
self.manager.prepare_store(resumed_blocks, ctx)
|
||||
self.manager.complete_store(resumed_blocks, ctx, success=True)
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
|
||||
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_on_new_request_lifecycle(self, manager_setup):
|
||||
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
|
||||
and is cleaned up on on_request_finished."""
|
||||
@@ -535,23 +714,25 @@ class TestTieringOffloadingManager:
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.BLOCK_LEVEL
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
# Escalate: tier1 requests REQUEST_LEVEL
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle_2")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.REQUEST_LEVEL
|
||||
assert ctx.req_id in self.manager._request_level_tiers
|
||||
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
|
||||
self.secondary_tier1
|
||||
}
|
||||
|
||||
# Cleanup
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._request_level_tiers
|
||||
assert ctx.req_id not in self.manager._req_state
|
||||
|
||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
||||
self, manager_setup
|
||||
@@ -559,6 +740,7 @@ class TestTieringOffloadingManager:
|
||||
"""prepare_store cascades hit blocks to request-level tiers only."""
|
||||
# Store some blocks to primary first
|
||||
existing_blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
result = self.manager.prepare_store(existing_blocks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(existing_blocks, _CTX, success=True)
|
||||
@@ -566,10 +748,8 @@ class TestTieringOffloadingManager:
|
||||
self._simulate_on_schedule_end()
|
||||
|
||||
# Make tier1 request-level, tier2 stays block-level
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_cascade")
|
||||
@@ -599,14 +779,15 @@ class TestTieringOffloadingManager:
|
||||
# tier2 (block-level) does not get existing blocks here.
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
def test_reset_cache_clears_all_state(self, manager_setup):
|
||||
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
|
||||
"""reset_cache wipes every kind of orchestrator state and resets
|
||||
primary tier; pending submissions are dropped without being sent
|
||||
to the secondary tier."""
|
||||
to the secondary tier. Active request state is retained."""
|
||||
# Cascade — populates primary blocks and leaves cascade jobs
|
||||
# in _transfer_jobs (the synchronous example tier has already
|
||||
# queued completions); reset_cache's drain loop will pick them up.
|
||||
blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
assert self.manager._transfer_jobs
|
||||
@@ -619,13 +800,14 @@ class TestTieringOffloadingManager:
|
||||
assert self.manager._pending_load_submissions
|
||||
|
||||
# Request-level tier registration.
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
self.secondary_tier1.on_new_request = lambda req_context: (
|
||||
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
|
||||
)
|
||||
self.manager.on_new_request(ReqContext(req_id="rl"))
|
||||
assert self.manager._request_level_tiers
|
||||
rl_ctx = ReqContext(req_id="rl")
|
||||
self.manager.on_new_request(rl_ctx)
|
||||
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
|
||||
self.secondary_tier1
|
||||
}
|
||||
|
||||
# Mark this step as already polled (reset_cache must clear it).
|
||||
self.manager._processed_jobs_this_step = True
|
||||
@@ -640,7 +822,7 @@ class TestTieringOffloadingManager:
|
||||
# Orchestrator state cleared.
|
||||
assert self.manager._transfer_jobs == {}
|
||||
assert self.manager._pending_load_submissions == {}
|
||||
assert self.manager._request_level_tiers == {}
|
||||
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
|
||||
assert self.manager._processed_jobs_this_step is False
|
||||
|
||||
# Primary tier reset to a fresh state.
|
||||
@@ -668,6 +850,7 @@ class TestTieringOffloadingManager:
|
||||
|
||||
# Drive a cascade so a job lands in _transfer_jobs.
|
||||
blocks = to_keys(range(3))
|
||||
self._start_request()
|
||||
self.manager.prepare_store(blocks, _CTX)
|
||||
self.manager.complete_store(blocks, _CTX, success=True)
|
||||
assert self.manager._transfer_jobs
|
||||
@@ -696,6 +879,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
blocks = to_keys(range(3))
|
||||
|
||||
# Should work like a regular OffloadingManager
|
||||
manager.on_new_request(_CTX)
|
||||
result = manager.prepare_store(blocks, _CTX)
|
||||
assert result is not None
|
||||
manager.complete_store(blocks, _CTX, success=True)
|
||||
|
||||
@@ -1140,11 +1140,6 @@ class OffloadingConnectorScheduler:
|
||||
del self._jobs[job_id]
|
||||
req_status.transfer_jobs.remove(job_id)
|
||||
if not req_status.transfer_jobs and req_status.req.is_finished():
|
||||
# Deferred from request_finished: the request's last in-flight
|
||||
# job is now done, so fire the finalize hook here, after the
|
||||
# final complete_store/complete_load above (and any submit_store
|
||||
# the complete_store cascade issued).
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
del self._req_status[job_status.req_id]
|
||||
|
||||
def get_stats(self) -> OffloadingConnectorStats | None:
|
||||
@@ -1181,20 +1176,23 @@ class OffloadingConnectorScheduler:
|
||||
if req_status is None:
|
||||
# Untracked request (offloading never started): no in-flight jobs,
|
||||
# nothing was deferred, so finalize immediately.
|
||||
self.manager.on_request_finished(_create_req_context(request))
|
||||
req_context = _create_req_context(request)
|
||||
self.manager.on_new_request(req_context)
|
||||
self.manager.on_request_finished(req_context)
|
||||
return False, None
|
||||
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
|
||||
if not req_status.transfer_jobs:
|
||||
# No in-flight jobs: all per-request calls are done, finalize now.
|
||||
self.manager.on_request_finished(req_status.req_context)
|
||||
# No in-flight jobs: no later complete_store()/complete_load() calls
|
||||
# need this request's state.
|
||||
del self._req_status[request.request_id]
|
||||
return False, None
|
||||
|
||||
# In-flight jobs remain, so defer on_request_finished to
|
||||
# update_connector_output, which fires it once the last job completes
|
||||
# (after the final complete_store and any cascade submit_store it
|
||||
# issues). These pending stores outlive the request's block ownership;
|
||||
# register them so future reuse of those blocks triggers a flush.
|
||||
# In-flight jobs remain after the request stopped. Their completion may
|
||||
# still call manager.complete_store()/complete_load(), so keep req_status.
|
||||
# Pending stores outlive the request's block ownership; register them so
|
||||
# future reuse of those blocks triggers a flush.
|
||||
for job_id in req_status.transfer_jobs:
|
||||
job_status = self._jobs[job_id]
|
||||
for bid in job_status.non_sliding_window_block_ids or ():
|
||||
@@ -1225,14 +1223,8 @@ class OffloadingConnectorScheduler:
|
||||
# Flush all in-flight jobs
|
||||
self._current_batch_jobs_to_flush.update(self._jobs.keys())
|
||||
|
||||
# A finished request may still be tracked here with in-flight jobs that
|
||||
# this reset discards, so its deferred on_request_finished() would never
|
||||
# fire (completions are skipped as stale) and its _req_status entry would
|
||||
# leak. Finalize such requests now, before resetting the manager.
|
||||
# list() snapshots because we delete while iterating.
|
||||
for req_id, status in list(self._req_status.items()):
|
||||
if status.req.is_finished():
|
||||
self.manager.on_request_finished(status.req_context)
|
||||
del self._req_status[req_id]
|
||||
|
||||
# Reset offloading manager cache
|
||||
|
||||
@@ -276,16 +276,16 @@ class OffloadingManager(ABC):
|
||||
"""
|
||||
Called when a request has finished.
|
||||
|
||||
By the time this is called, all per-request offload calls for this
|
||||
request (prepare_store/complete_store, prepare_load/complete_load,
|
||||
touch, lookup) have already been issued, and none will follow. The
|
||||
scheduler defers this call until the request is finished and has no
|
||||
in-flight transfer jobs.
|
||||
By the time this is called, the scheduler will issue no more
|
||||
submit-side calls for this request, such as prepare_store() and
|
||||
prepare_load(). Completion callbacks for already-submitted transfers
|
||||
(complete_store() and complete_load()) may still arrive afterward.
|
||||
|
||||
Note this signals only that no further calls will be made; it does NOT
|
||||
imply the data has been persisted. Asynchronous transfers already
|
||||
submitted for this request (e.g. CPU->secondary cascades) may still be
|
||||
in flight. This is the right place to release per-request bookkeeping.
|
||||
This hook does NOT imply the data has been persisted. Asynchronous
|
||||
transfers already submitted for this request may still be in flight.
|
||||
Managers that cascade to lower tiers should delay those tiers'
|
||||
on_request_finished() calls until no more lower-tier submit calls can
|
||||
be issued for this request.
|
||||
|
||||
Args:
|
||||
req_context: per-request context.
|
||||
|
||||
@@ -20,7 +20,6 @@ Key Design Principles:
|
||||
protecting blocks from eviction until complete_read() is called
|
||||
"""
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Collection, Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
@@ -62,6 +61,14 @@ class PendingPromotion:
|
||||
block_ids: list[int] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RequestState:
|
||||
req_context: ReqContext
|
||||
pending_primary_stores: int = 0
|
||||
is_finished: bool = False
|
||||
request_level_tiers: set[SecondaryTierManager] | None = None
|
||||
|
||||
|
||||
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
|
||||
"""CPUOffloadingManager with a primary/secondary transfer interface.
|
||||
|
||||
@@ -166,12 +173,10 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# Reset at the end of each step in on_schedule_end().
|
||||
self._processed_jobs_this_step: bool = False
|
||||
|
||||
# Per-request set of secondary tiers that requested REQUEST_LEVEL
|
||||
# policy. Populated in on_new_request(),
|
||||
# cleaned up in on_request_finished().
|
||||
self._request_level_tiers: defaultdict[str, set[SecondaryTierManager]] = (
|
||||
defaultdict(set)
|
||||
)
|
||||
# Per-request state for prepared GPU->primary stores and finalization.
|
||||
# Secondary tiers are finalized only after pending primary stores reach
|
||||
# complete_store(), since complete_store() can still submit cascades.
|
||||
self._req_state: dict[str, RequestState] = {}
|
||||
|
||||
def _next_job_id(self) -> JobId:
|
||||
"""Generate a unique job ID for async transfer tracking."""
|
||||
@@ -432,9 +437,13 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
if primary_result is None:
|
||||
return None
|
||||
|
||||
if primary_result.keys_to_store:
|
||||
state = self._req_state[req_context.req_id]
|
||||
state.pending_primary_stores += 1
|
||||
|
||||
# Step 3: For request-level tiers, cascade blocks already in primary
|
||||
request_level_tiers = self._request_level_tiers.get(req_context.req_id)
|
||||
if request_level_tiers is not None:
|
||||
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
|
||||
if request_level_tiers:
|
||||
keys_to_store_set = set(primary_result.keys_to_store)
|
||||
keys_already_in_primary = tuple(
|
||||
k for k in keys if k not in keys_to_store_set
|
||||
@@ -508,36 +517,38 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# Step 1: Complete store in primary tier (makes blocks loadable)
|
||||
self.primary_tier.complete_store(keys, req_context, success)
|
||||
|
||||
if not success:
|
||||
# If GPU→Primary transfer failed, don't cascade to secondary tiers
|
||||
return
|
||||
if success:
|
||||
# Step 2: Cascade to ALL secondary tiers
|
||||
# For each secondary tier, call primary.prepare_read() to get the
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier in self.secondary_tiers:
|
||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
|
||||
# Step 2: Cascade to ALL secondary tiers
|
||||
# For each secondary tier, call primary.prepare_read() to get the
|
||||
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
|
||||
# eviction during the async transfer). One prepare_read() call per
|
||||
# secondary tier.
|
||||
for tier in self.secondary_tiers:
|
||||
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
|
||||
# Submit async store job: primary→secondary
|
||||
job_id = self._next_job_id()
|
||||
|
||||
# Submit async store job: primary→secondary
|
||||
job_id = self._next_job_id()
|
||||
# Track this store job
|
||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||
job_metadata = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=primary_blocks_spec.block_ids,
|
||||
is_promotion=False,
|
||||
req_context=req_context,
|
||||
)
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
|
||||
# Track this store job
|
||||
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
|
||||
job_metadata = JobMetadata(
|
||||
job_id=job_id,
|
||||
keys=keys,
|
||||
block_ids=primary_blocks_spec.block_ids,
|
||||
is_promotion=False,
|
||||
req_context=req_context,
|
||||
)
|
||||
self._transfer_jobs[job_id] = job_metadata
|
||||
|
||||
tier.submit_store(job_metadata)
|
||||
tier.submit_store(job_metadata)
|
||||
|
||||
# Note: The async transfers are now in flight. Their completion is
|
||||
# tracked via get_finished_jobs() / _maybe_process_finished_jobs().
|
||||
req_id = req_context.req_id
|
||||
state = self._req_state[req_id]
|
||||
assert state.pending_primary_stores > 0
|
||||
state.pending_primary_stores -= 1
|
||||
self._maybe_finalize_request(req_id)
|
||||
|
||||
@override
|
||||
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
|
||||
@@ -547,14 +558,18 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
Returns REQUEST_LEVEL if ANY secondary tier wants request-level.
|
||||
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
|
||||
"""
|
||||
state = RequestState(req_context=req_context)
|
||||
for tier in self.secondary_tiers:
|
||||
tier_ctx = tier.on_new_request(req_context)
|
||||
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
|
||||
self._request_level_tiers[req_context.req_id].add(tier)
|
||||
if state.request_level_tiers is None:
|
||||
state.request_level_tiers = set()
|
||||
state.request_level_tiers.add(tier)
|
||||
self._req_state[req_context.req_id] = state
|
||||
|
||||
policy = (
|
||||
OffloadPolicy.REQUEST_LEVEL
|
||||
if req_context.req_id in self._request_level_tiers
|
||||
if state.request_level_tiers
|
||||
else OffloadPolicy.BLOCK_LEVEL
|
||||
)
|
||||
return RequestOffloadingContext(policy=policy)
|
||||
@@ -562,9 +577,26 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
@override
|
||||
def on_request_finished(self, req_context: ReqContext) -> None:
|
||||
self.primary_tier.on_request_finished(req_context)
|
||||
state = self._req_state[req_context.req_id]
|
||||
state.is_finished = True
|
||||
self._maybe_finalize_request(req_context.req_id)
|
||||
|
||||
def _maybe_finalize_request(self, req_id: str) -> None:
|
||||
"""Finalize secondary tiers once no more store cascades can be submitted.
|
||||
|
||||
Finalization means forwarding on_request_finished() to secondary tiers.
|
||||
It is delayed until pending GPU->primary stores finish, since their
|
||||
complete_store() callbacks may still submit primary->secondary stores.
|
||||
"""
|
||||
state = self._req_state[req_id]
|
||||
if not state.is_finished:
|
||||
return
|
||||
if state.pending_primary_stores != 0:
|
||||
return
|
||||
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_request_finished(req_context)
|
||||
self._request_level_tiers.pop(req_context.req_id, None)
|
||||
tier.on_request_finished(state.req_context)
|
||||
del self._req_state[req_id]
|
||||
|
||||
@override
|
||||
def on_schedule_end(self) -> None:
|
||||
@@ -604,7 +636,7 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
|
||||
@override
|
||||
def reset_cache(self) -> None:
|
||||
"""Drop all tracked state in the orchestrator and primary tier.
|
||||
"""Reset transfer bookkeeping and primary-tier cache.
|
||||
|
||||
Called during sleep, weight update, or resume. Each secondary tier
|
||||
drains its in-flight transfers via drain_jobs() so no tier I/O is
|
||||
@@ -613,7 +645,9 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
from reusing primary slots while a transfer is mid-copy.
|
||||
|
||||
Secondary tiers are intentionally not reset: persistent stores
|
||||
(FS, network) keep their data across resets.
|
||||
(FS, network) keep their data across resets. Active request state is
|
||||
retained so those requests can continue after the reset; finished
|
||||
requests are finalized and removed.
|
||||
"""
|
||||
for tier in self.secondary_tiers:
|
||||
tier.drain_jobs()
|
||||
@@ -626,9 +660,19 @@ class TieringOffloadingManager(OffloadingManager):
|
||||
# called so no tier I/O is touching that memory.
|
||||
self._pending_load_submissions.clear()
|
||||
|
||||
finished_req_ids = []
|
||||
for req_id, state in self._req_state.items():
|
||||
state.pending_primary_stores = 0
|
||||
if not state.is_finished:
|
||||
continue
|
||||
for tier in self.secondary_tiers:
|
||||
tier.on_request_finished(state.req_context)
|
||||
finished_req_ids.append(req_id)
|
||||
|
||||
self.primary_tier.reset_cache()
|
||||
|
||||
self._request_level_tiers.clear()
|
||||
for req_id in finished_req_ids:
|
||||
del self._req_state[req_id]
|
||||
self._processed_jobs_this_step = False
|
||||
|
||||
@override
|
||||
|
||||
Reference in New Issue
Block a user