diff --git a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py index e4e5c50ecd..32abd05242 100644 --- a/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py +++ b/tests/v1/kv_connector/unit/offloading_connector/test_scheduler.py @@ -216,17 +216,14 @@ def test_request_preemption(request_runner, async_scheduling: bool): @pytest.mark.parametrize("async_scheduling", [True, False]) -def test_no_offload_call_after_on_request_finished( +def test_on_request_finished_is_not_deferred_until_store_completion( request_runner, async_scheduling: bool ): - """on_request_finished is not issued before a per-request offload - call. + """on_request_finished fires when no more stores will be submitted. - A request can finish while its GPU->primary store is still in flight; the - later worker completion then drives complete_store. The scheduler defers - on_request_finished until the request is finished AND has no in-flight - transfer jobs, so complete_store is observed BEFORE on_request_finished, - and it is called exactly once. + A request can finish while its GPU->primary store is still in flight. The + manager-level hook should not wait for that completion; complete_store may + still arrive afterward for already-submitted transfer jobs. """ block_size = 4 block_size_factor = 3 @@ -263,27 +260,34 @@ def test_no_offload_call_after_on_request_finished( complete_transfers=False, ) - # Finish the request, completing its pending stores. on_request_finished is - # deferred until the stores drain, so it lands after the last complete_store. - # 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor - # GPU blocks. + # Finish the request while its stores are still in flight. The hook should + # fire immediately even though no complete_store has arrived yet. runner.run( decoded_tokens=[EOS_TOKEN_ID], - expected_stored=tuple(range(4 * block_size_factor)), + complete_transfers=False, ) req_id = str(runner.req_id) + assert calls == [("on_request_finished", req_id)], calls + + # Drain the stores afterward. The already-submitted complete_store calls + # are allowed to arrive after on_request_finished. + runner.run( + decoded_tokens=[], + complete_transfers=True, + expected_stored=tuple(range(4 * block_size_factor)), + ) + # on_request_finished is issued exactly once. assert calls.count(("on_request_finished", req_id)) == 1, calls finished_idx = calls.index(("on_request_finished", req_id)) store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)] - # All of the request's complete_store calls must precede its single - # on_request_finished. + # The request-level hook no longer waits for already-submitted transfers. assert store_indices, calls - assert max(store_indices) < finished_idx, calls + assert finished_idx < min(store_indices), calls @pytest.mark.parametrize("async_scheduling", [True, False]) @@ -1232,11 +1236,8 @@ def test_reset_cache(request_runner, async_scheduling: bool): def test_reset_cache_finalizes_finished_request_with_pending_store( request_runner, async_scheduling: bool ): - """reset_cache must finalize a finished request whose in-flight stores it - discards: call on_request_finished and drop its _req_status entry. - - Otherwise the deferred hook (which waits for the now-discarded jobs to - complete) never fires and the entry leaks. + """reset_cache drops a finished request whose in-flight stores it discards + without calling on_request_finished twice. """ block_size = 4 block_size_factor = 3 @@ -1273,14 +1274,15 @@ def test_reset_cache_finalizes_finished_request_with_pending_store( assert any(job.is_store for job in cs._jobs.values()) # Finish the request while its store is still in flight. request_finished - # takes the defer branch (pending jobs), so on_request_finished is NOT - # called yet and the entry stays tracked. + # fires the hook eagerly, but the entry stays tracked so later completions + # can still call complete_store(). req_status.req.status = RequestStatus.FINISHED_STOPPED cs.request_finished(req_status.req) - assert finalized == [] + assert finalized == [req_id] assert req_id in cs._req_status - # reset_cache discards the in-flight store; it must finalize the request. + # reset_cache discards the in-flight store and drops the state without a + # duplicate on_request_finished call. cs.reset_cache() assert finalized == [req_id] assert req_id not in cs._req_status diff --git a/tests/v1/kv_offload/tiering/test_tiering_offloading.py b/tests/v1/kv_offload/tiering/test_tiering_offloading.py index b1b4df5363..fca8453244 100644 --- a/tests/v1/kv_offload/tiering/test_tiering_offloading.py +++ b/tests/v1/kv_offload/tiering/test_tiering_offloading.py @@ -235,11 +235,16 @@ class TestTieringOffloadingManager: self.manager.on_schedule_end() list(self.manager.take_events()) + def _start_request(self, req_context: ReqContext = _CTX): + if req_context.req_id not in self.manager._req_state: + self.manager.on_new_request(req_context) + def test_basic_store_to_primary(self, manager_setup): """Test basic store operation to primary tier.""" blocks = to_keys(range(3)) # Prepare store + self._start_request() result = self.manager.prepare_store(blocks, _CTX) assert result is not None assert len(result.keys_to_store) == 3 @@ -262,6 +267,7 @@ class TestTieringOffloadingManager: ) # Store to primary + self._start_request() result = self.manager.prepare_store(blocks, _CTX) assert result is not None @@ -285,6 +291,7 @@ class TestTieringOffloadingManager: blocks = to_keys(range(3)) # Store to primary + self._start_request() result = self.manager.prepare_store(blocks, _CTX) assert result is not None self.manager.complete_store(blocks, _CTX, success=True) @@ -330,6 +337,7 @@ class TestTieringOffloadingManager: blocks = to_keys(range(3)) # Store blocks + self._start_request() self.manager.prepare_store(blocks, _CTX) self.manager.complete_store(blocks, _CTX, success=True) @@ -366,6 +374,7 @@ class TestTieringOffloadingManager: blocks = to_keys(range(5)) # Store first 3 blocks to primary + self._start_request() self.manager.prepare_store(blocks[:3], _CTX) self.manager.complete_store(blocks[:3], _CTX, success=True) @@ -377,6 +386,7 @@ class TestTieringOffloadingManager: # Primary tier has capacity of 5 blocks # First, fill the primary tier blocks = to_keys(range(5)) + self._start_request() result = self.manager.prepare_store(blocks, _CTX) assert result is not None assert len(result.keys_to_store) == 5 @@ -399,6 +409,7 @@ class TestTieringOffloadingManager: blocks = to_keys(range(3)) # Store blocks + self._start_request() self.manager.prepare_store(blocks, _CTX) self.manager.complete_store(blocks, _CTX, success=True) self._simulate_on_schedule_end() @@ -431,6 +442,7 @@ class TestTieringOffloadingManager: ) # Prepare store + self._start_request() result = self.manager.prepare_store(blocks, _CTX) assert result is not None @@ -521,6 +533,7 @@ class TestTieringOffloadingManager: ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"}) + self._start_request(ctx) self.manager.prepare_store(blocks, ctx) self.manager.complete_store(blocks, ctx, success=True) @@ -528,6 +541,172 @@ class TestTieringOffloadingManager: job_metadata = self.secondary_tier1.submit_store.call_args.args[0] assert job_metadata.req_context is ctx + def test_on_request_finished_delays_secondary_until_store_submitted( + self, manager_setup + ): + """Manager hook is eager; secondary hooks wait for cascade submission.""" + blocks = to_keys(range(2)) + ctx = ReqContext(req_id="req_delayed_secondary") + calls: list[tuple[str, str]] = [] + + self.primary_tier.on_request_finished = MagicMock( + side_effect=lambda req_context: calls.append( + ("primary_finish", req_context.req_id) + ) + ) + + original_submit_store1 = self.secondary_tier1.submit_store + original_submit_store2 = self.secondary_tier2.submit_store + + def submit_store1(job_metadata): + calls.append(("submit_store_1", job_metadata.req_context.req_id)) + return original_submit_store1(job_metadata) + + def submit_store2(job_metadata): + calls.append(("submit_store_2", job_metadata.req_context.req_id)) + return original_submit_store2(job_metadata) + + self.secondary_tier1.submit_store = MagicMock(side_effect=submit_store1) + self.secondary_tier2.submit_store = MagicMock(side_effect=submit_store2) + self.secondary_tier1.on_request_finished = MagicMock( + side_effect=lambda req_context: calls.append( + ("secondary_finish_1", req_context.req_id) + ) + ) + self.secondary_tier2.on_request_finished = MagicMock( + side_effect=lambda req_context: calls.append( + ("secondary_finish_2", req_context.req_id) + ) + ) + + self._start_request(ctx) + self.manager.prepare_store(blocks, ctx) + self.manager.on_request_finished(ctx) + + assert calls == [("primary_finish", ctx.req_id)] + self.secondary_tier1.on_request_finished.assert_not_called() + self.secondary_tier2.on_request_finished.assert_not_called() + + self.manager.complete_store(blocks, ctx, success=True) + + assert calls == [ + ("primary_finish", ctx.req_id), + ("submit_store_1", ctx.req_id), + ("submit_store_2", ctx.req_id), + ("secondary_finish_1", ctx.req_id), + ("secondary_finish_2", ctx.req_id), + ] + + def test_failed_store_finalizes_finished_request(self, manager_setup): + """Failed primary stores still unblock secondary finalization.""" + blocks = to_keys(range(2)) + ctx = ReqContext(req_id="req_failed_store_finalize") + + self.secondary_tier1.submit_store = MagicMock( + wraps=self.secondary_tier1.submit_store + ) + self.secondary_tier2.submit_store = MagicMock( + wraps=self.secondary_tier2.submit_store + ) + self.secondary_tier1.on_request_finished = MagicMock( + wraps=self.secondary_tier1.on_request_finished + ) + self.secondary_tier2.on_request_finished = MagicMock( + wraps=self.secondary_tier2.on_request_finished + ) + + self._start_request(ctx) + self.manager.prepare_store(blocks, ctx) + self.manager.on_request_finished(ctx) + + self.secondary_tier1.on_request_finished.assert_not_called() + self.secondary_tier2.on_request_finished.assert_not_called() + + self.manager.complete_store(blocks, ctx, success=False) + + self.secondary_tier1.submit_store.assert_not_called() + self.secondary_tier2.submit_store.assert_not_called() + self.secondary_tier1.on_request_finished.assert_called_once_with(ctx) + self.secondary_tier2.on_request_finished.assert_called_once_with(ctx) + assert ctx.req_id not in self.manager._req_state + + def test_zero_store_request_finalizes_immediately(self, manager_setup): + """Requests with no pending stores finalize secondary tiers immediately.""" + ctx = ReqContext(req_id="req_zero_store_finalize") + + self.secondary_tier1.on_request_finished = MagicMock( + wraps=self.secondary_tier1.on_request_finished + ) + self.secondary_tier2.on_request_finished = MagicMock( + wraps=self.secondary_tier2.on_request_finished + ) + + self._start_request(ctx) + self.manager.on_request_finished(ctx) + + self.secondary_tier1.on_request_finished.assert_called_once_with(ctx) + self.secondary_tier2.on_request_finished.assert_called_once_with(ctx) + assert ctx.req_id not in self.manager._req_state + + def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup): + """reset_cache abandons pending primary stores and finalizes secondaries.""" + blocks = to_keys(range(2)) + ctx = ReqContext(req_id="req_reset_finalize_secondary") + + self.secondary_tier1.on_request_finished = MagicMock( + wraps=self.secondary_tier1.on_request_finished + ) + self.secondary_tier2.on_request_finished = MagicMock( + wraps=self.secondary_tier2.on_request_finished + ) + + self._start_request(ctx) + self.manager.prepare_store(blocks, ctx) + self.manager.on_request_finished(ctx) + + self.secondary_tier1.on_request_finished.assert_not_called() + self.secondary_tier2.on_request_finished.assert_not_called() + + self.manager.reset_cache() + + self.secondary_tier1.on_request_finished.assert_called_once_with(ctx) + self.secondary_tier2.on_request_finished.assert_called_once_with(ctx) + assert self.manager._req_state == {} + + def test_reset_cache_clears_pending_primary_stores_for_active_request( + self, manager_setup + ): + """reset_cache drops active pending stores so resumed requests finalize.""" + initial_blocks = to_keys(range(2)) + resumed_blocks = to_keys(range(2, 4)) + ctx = ReqContext(req_id="req_reset_resume") + + self.secondary_tier1.on_request_finished = MagicMock( + wraps=self.secondary_tier1.on_request_finished + ) + self.secondary_tier2.on_request_finished = MagicMock( + wraps=self.secondary_tier2.on_request_finished + ) + + self._start_request(ctx) + self.manager.prepare_store(initial_blocks, ctx) + assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1 + + self.manager.reset_cache() + + assert ctx.req_id in self.manager._req_state + assert self.manager._req_state[ctx.req_id].pending_primary_stores == 0 + self.secondary_tier1.on_request_finished.assert_not_called() + self.secondary_tier2.on_request_finished.assert_not_called() + + self.manager.prepare_store(resumed_blocks, ctx) + self.manager.complete_store(resumed_blocks, ctx, success=True) + self.manager.on_request_finished(ctx) + + self.secondary_tier1.on_request_finished.assert_called_once_with(ctx) + self.secondary_tier2.on_request_finished.assert_called_once_with(ctx) + assert ctx.req_id not in self.manager._req_state + def test_on_new_request_lifecycle(self, manager_setup): """Policy defaults to BLOCK_LEVEL, escalates when a tier requests it, and is cleaned up on on_request_finished.""" @@ -535,23 +714,25 @@ class TestTieringOffloadingManager: ctx = ReqContext(req_id="req_policy_lifecycle") result = self.manager.on_new_request(ctx) assert result.policy == OffloadPolicy.BLOCK_LEVEL + assert self.manager._req_state[ctx.req_id].request_level_tiers is None self.manager.on_request_finished(ctx) + assert ctx.req_id not in self.manager._req_state # Escalate: tier1 requests REQUEST_LEVEL - self.secondary_tier1.on_new_request = ( - lambda req_context: RequestOffloadingContext( - policy=OffloadPolicy.REQUEST_LEVEL - ) + self.secondary_tier1.on_new_request = lambda req_context: ( + RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL) ) ctx = ReqContext(req_id="req_policy_lifecycle_2") result = self.manager.on_new_request(ctx) assert result.policy == OffloadPolicy.REQUEST_LEVEL - assert ctx.req_id in self.manager._request_level_tiers + assert self.manager._req_state[ctx.req_id].request_level_tiers == { + self.secondary_tier1 + } # Cleanup self.manager.on_request_finished(ctx) - assert ctx.req_id not in self.manager._request_level_tiers + assert ctx.req_id not in self.manager._req_state def test_prepare_store_cascades_existing_blocks_to_request_level_tiers( self, manager_setup @@ -559,6 +740,7 @@ class TestTieringOffloadingManager: """prepare_store cascades hit blocks to request-level tiers only.""" # Store some blocks to primary first existing_blocks = to_keys(range(3)) + self._start_request() result = self.manager.prepare_store(existing_blocks, _CTX) assert result is not None self.manager.complete_store(existing_blocks, _CTX, success=True) @@ -566,10 +748,8 @@ class TestTieringOffloadingManager: self._simulate_on_schedule_end() # Make tier1 request-level, tier2 stays block-level - self.secondary_tier1.on_new_request = ( - lambda req_context: RequestOffloadingContext( - policy=OffloadPolicy.REQUEST_LEVEL - ) + self.secondary_tier1.on_new_request = lambda req_context: ( + RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL) ) ctx = ReqContext(req_id="req_cascade") @@ -599,14 +779,15 @@ class TestTieringOffloadingManager: # tier2 (block-level) does not get existing blocks here. self.secondary_tier2.submit_store.assert_not_called() - def test_reset_cache_clears_all_state(self, manager_setup): + def test_reset_cache_clears_orchestrator_state(self, manager_setup): """reset_cache wipes every kind of orchestrator state and resets primary tier; pending submissions are dropped without being sent - to the secondary tier.""" + to the secondary tier. Active request state is retained.""" # Cascade — populates primary blocks and leaves cascade jobs # in _transfer_jobs (the synchronous example tier has already # queued completions); reset_cache's drain loop will pick them up. blocks = to_keys(range(3)) + self._start_request() self.manager.prepare_store(blocks, _CTX) self.manager.complete_store(blocks, _CTX, success=True) assert self.manager._transfer_jobs @@ -619,13 +800,14 @@ class TestTieringOffloadingManager: assert self.manager._pending_load_submissions # Request-level tier registration. - self.secondary_tier1.on_new_request = ( - lambda req_context: RequestOffloadingContext( - policy=OffloadPolicy.REQUEST_LEVEL - ) + self.secondary_tier1.on_new_request = lambda req_context: ( + RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL) ) - self.manager.on_new_request(ReqContext(req_id="rl")) - assert self.manager._request_level_tiers + rl_ctx = ReqContext(req_id="rl") + self.manager.on_new_request(rl_ctx) + assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == { + self.secondary_tier1 + } # Mark this step as already polled (reset_cache must clear it). self.manager._processed_jobs_this_step = True @@ -640,7 +822,7 @@ class TestTieringOffloadingManager: # Orchestrator state cleared. assert self.manager._transfer_jobs == {} assert self.manager._pending_load_submissions == {} - assert self.manager._request_level_tiers == {} + assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id} assert self.manager._processed_jobs_this_step is False # Primary tier reset to a fresh state. @@ -668,6 +850,7 @@ class TestTieringOffloadingManager: # Drive a cascade so a job lands in _transfer_jobs. blocks = to_keys(range(3)) + self._start_request() self.manager.prepare_store(blocks, _CTX) self.manager.complete_store(blocks, _CTX, success=True) assert self.manager._transfer_jobs @@ -696,6 +879,7 @@ class TestTieringOffloadingWithoutSecondaryTiers: blocks = to_keys(range(3)) # Should work like a regular OffloadingManager + manager.on_new_request(_CTX) result = manager.prepare_store(blocks, _CTX) assert result is not None manager.complete_store(blocks, _CTX, success=True) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py index 4965d9db87..a35970a616 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/offloading/scheduler.py @@ -1140,11 +1140,6 @@ class OffloadingConnectorScheduler: del self._jobs[job_id] req_status.transfer_jobs.remove(job_id) if not req_status.transfer_jobs and req_status.req.is_finished(): - # Deferred from request_finished: the request's last in-flight - # job is now done, so fire the finalize hook here, after the - # final complete_store/complete_load above (and any submit_store - # the complete_store cascade issued). - self.manager.on_request_finished(req_status.req_context) del self._req_status[job_status.req_id] def get_stats(self) -> OffloadingConnectorStats | None: @@ -1181,20 +1176,23 @@ class OffloadingConnectorScheduler: if req_status is None: # Untracked request (offloading never started): no in-flight jobs, # nothing was deferred, so finalize immediately. - self.manager.on_request_finished(_create_req_context(request)) + req_context = _create_req_context(request) + self.manager.on_new_request(req_context) + self.manager.on_request_finished(req_context) return False, None + self.manager.on_request_finished(req_status.req_context) + if not req_status.transfer_jobs: - # No in-flight jobs: all per-request calls are done, finalize now. - self.manager.on_request_finished(req_status.req_context) + # No in-flight jobs: no later complete_store()/complete_load() calls + # need this request's state. del self._req_status[request.request_id] return False, None - # In-flight jobs remain, so defer on_request_finished to - # update_connector_output, which fires it once the last job completes - # (after the final complete_store and any cascade submit_store it - # issues). These pending stores outlive the request's block ownership; - # register them so future reuse of those blocks triggers a flush. + # In-flight jobs remain after the request stopped. Their completion may + # still call manager.complete_store()/complete_load(), so keep req_status. + # Pending stores outlive the request's block ownership; register them so + # future reuse of those blocks triggers a flush. for job_id in req_status.transfer_jobs: job_status = self._jobs[job_id] for bid in job_status.non_sliding_window_block_ids or (): @@ -1225,14 +1223,8 @@ class OffloadingConnectorScheduler: # Flush all in-flight jobs self._current_batch_jobs_to_flush.update(self._jobs.keys()) - # A finished request may still be tracked here with in-flight jobs that - # this reset discards, so its deferred on_request_finished() would never - # fire (completions are skipped as stale) and its _req_status entry would - # leak. Finalize such requests now, before resetting the manager. - # list() snapshots because we delete while iterating. for req_id, status in list(self._req_status.items()): if status.req.is_finished(): - self.manager.on_request_finished(status.req_context) del self._req_status[req_id] # Reset offloading manager cache diff --git a/vllm/v1/kv_offload/base.py b/vllm/v1/kv_offload/base.py index 70a53b072f..db0940ef38 100644 --- a/vllm/v1/kv_offload/base.py +++ b/vllm/v1/kv_offload/base.py @@ -276,16 +276,16 @@ class OffloadingManager(ABC): """ Called when a request has finished. - By the time this is called, all per-request offload calls for this - request (prepare_store/complete_store, prepare_load/complete_load, - touch, lookup) have already been issued, and none will follow. The - scheduler defers this call until the request is finished and has no - in-flight transfer jobs. + By the time this is called, the scheduler will issue no more + submit-side calls for this request, such as prepare_store() and + prepare_load(). Completion callbacks for already-submitted transfers + (complete_store() and complete_load()) may still arrive afterward. - Note this signals only that no further calls will be made; it does NOT - imply the data has been persisted. Asynchronous transfers already - submitted for this request (e.g. CPU->secondary cascades) may still be - in flight. This is the right place to release per-request bookkeeping. + This hook does NOT imply the data has been persisted. Asynchronous + transfers already submitted for this request may still be in flight. + Managers that cascade to lower tiers should delay those tiers' + on_request_finished() calls until no more lower-tier submit calls can + be issued for this request. Args: req_context: per-request context. diff --git a/vllm/v1/kv_offload/tiering/manager.py b/vllm/v1/kv_offload/tiering/manager.py index abed61a3e4..85346eacb7 100644 --- a/vllm/v1/kv_offload/tiering/manager.py +++ b/vllm/v1/kv_offload/tiering/manager.py @@ -20,7 +20,6 @@ Key Design Principles: protecting blocks from eviction until complete_read() is called """ -from collections import defaultdict from collections.abc import Collection, Iterable, Sequence from dataclasses import dataclass, field @@ -62,6 +61,14 @@ class PendingPromotion: block_ids: list[int] = field(default_factory=list) +@dataclass(slots=True) +class RequestState: + req_context: ReqContext + pending_primary_stores: int = 0 + is_finished: bool = False + request_level_tiers: set[SecondaryTierManager] | None = None + + class CPUPrimaryTierOffloadingManager(CPUOffloadingManager): """CPUOffloadingManager with a primary/secondary transfer interface. @@ -166,12 +173,10 @@ class TieringOffloadingManager(OffloadingManager): # Reset at the end of each step in on_schedule_end(). self._processed_jobs_this_step: bool = False - # Per-request set of secondary tiers that requested REQUEST_LEVEL - # policy. Populated in on_new_request(), - # cleaned up in on_request_finished(). - self._request_level_tiers: defaultdict[str, set[SecondaryTierManager]] = ( - defaultdict(set) - ) + # Per-request state for prepared GPU->primary stores and finalization. + # Secondary tiers are finalized only after pending primary stores reach + # complete_store(), since complete_store() can still submit cascades. + self._req_state: dict[str, RequestState] = {} def _next_job_id(self) -> JobId: """Generate a unique job ID for async transfer tracking.""" @@ -432,9 +437,13 @@ class TieringOffloadingManager(OffloadingManager): if primary_result is None: return None + if primary_result.keys_to_store: + state = self._req_state[req_context.req_id] + state.pending_primary_stores += 1 + # Step 3: For request-level tiers, cascade blocks already in primary - request_level_tiers = self._request_level_tiers.get(req_context.req_id) - if request_level_tiers is not None: + request_level_tiers = self._req_state[req_context.req_id].request_level_tiers + if request_level_tiers: keys_to_store_set = set(primary_result.keys_to_store) keys_already_in_primary = tuple( k for k in keys if k not in keys_to_store_set @@ -508,36 +517,38 @@ class TieringOffloadingManager(OffloadingManager): # Step 1: Complete store in primary tier (makes blocks loadable) self.primary_tier.complete_store(keys, req_context, success) - if not success: - # If GPU→Primary transfer failed, don't cascade to secondary tiers - return + if success: + # Step 2: Cascade to ALL secondary tiers + # For each secondary tier, call primary.prepare_read() to get the + # LoadStoreSpec AND to increment ref_cnt (protecting blocks from + # eviction during the async transfer). One prepare_read() call per + # secondary tier. + for tier in self.secondary_tiers: + primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context) - # Step 2: Cascade to ALL secondary tiers - # For each secondary tier, call primary.prepare_read() to get the - # LoadStoreSpec AND to increment ref_cnt (protecting blocks from - # eviction during the async transfer). One prepare_read() call per - # secondary tier. - for tier in self.secondary_tiers: - primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context) + # Submit async store job: primary→secondary + job_id = self._next_job_id() - # Submit async store job: primary→secondary - job_id = self._next_job_id() + # Track this store job + assert isinstance(primary_blocks_spec, CPULoadStoreSpec) + job_metadata = JobMetadata( + job_id=job_id, + keys=keys, + block_ids=primary_blocks_spec.block_ids, + is_promotion=False, + req_context=req_context, + ) + self._transfer_jobs[job_id] = job_metadata - # Track this store job - assert isinstance(primary_blocks_spec, CPULoadStoreSpec) - job_metadata = JobMetadata( - job_id=job_id, - keys=keys, - block_ids=primary_blocks_spec.block_ids, - is_promotion=False, - req_context=req_context, - ) - self._transfer_jobs[job_id] = job_metadata - - tier.submit_store(job_metadata) + tier.submit_store(job_metadata) # Note: The async transfers are now in flight. Their completion is # tracked via get_finished_jobs() / _maybe_process_finished_jobs(). + req_id = req_context.req_id + state = self._req_state[req_id] + assert state.pending_primary_stores > 0 + state.pending_primary_stores -= 1 + self._maybe_finalize_request(req_id) @override def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext: @@ -547,14 +558,18 @@ class TieringOffloadingManager(OffloadingManager): Returns REQUEST_LEVEL if ANY secondary tier wants request-level. Only stores REQUEST_LEVEL tier decisions for use in prepare_store. """ + state = RequestState(req_context=req_context) for tier in self.secondary_tiers: tier_ctx = tier.on_new_request(req_context) if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL: - self._request_level_tiers[req_context.req_id].add(tier) + if state.request_level_tiers is None: + state.request_level_tiers = set() + state.request_level_tiers.add(tier) + self._req_state[req_context.req_id] = state policy = ( OffloadPolicy.REQUEST_LEVEL - if req_context.req_id in self._request_level_tiers + if state.request_level_tiers else OffloadPolicy.BLOCK_LEVEL ) return RequestOffloadingContext(policy=policy) @@ -562,9 +577,26 @@ class TieringOffloadingManager(OffloadingManager): @override def on_request_finished(self, req_context: ReqContext) -> None: self.primary_tier.on_request_finished(req_context) + state = self._req_state[req_context.req_id] + state.is_finished = True + self._maybe_finalize_request(req_context.req_id) + + def _maybe_finalize_request(self, req_id: str) -> None: + """Finalize secondary tiers once no more store cascades can be submitted. + + Finalization means forwarding on_request_finished() to secondary tiers. + It is delayed until pending GPU->primary stores finish, since their + complete_store() callbacks may still submit primary->secondary stores. + """ + state = self._req_state[req_id] + if not state.is_finished: + return + if state.pending_primary_stores != 0: + return + for tier in self.secondary_tiers: - tier.on_request_finished(req_context) - self._request_level_tiers.pop(req_context.req_id, None) + tier.on_request_finished(state.req_context) + del self._req_state[req_id] @override def on_schedule_end(self) -> None: @@ -604,7 +636,7 @@ class TieringOffloadingManager(OffloadingManager): @override def reset_cache(self) -> None: - """Drop all tracked state in the orchestrator and primary tier. + """Reset transfer bookkeeping and primary-tier cache. Called during sleep, weight update, or resume. Each secondary tier drains its in-flight transfers via drain_jobs() so no tier I/O is @@ -613,7 +645,9 @@ class TieringOffloadingManager(OffloadingManager): from reusing primary slots while a transfer is mid-copy. Secondary tiers are intentionally not reset: persistent stores - (FS, network) keep their data across resets. + (FS, network) keep their data across resets. Active request state is + retained so those requests can continue after the reset; finished + requests are finalized and removed. """ for tier in self.secondary_tiers: tier.drain_jobs() @@ -626,9 +660,19 @@ class TieringOffloadingManager(OffloadingManager): # called so no tier I/O is touching that memory. self._pending_load_submissions.clear() + finished_req_ids = [] + for req_id, state in self._req_state.items(): + state.pending_primary_stores = 0 + if not state.is_finished: + continue + for tier in self.secondary_tiers: + tier.on_request_finished(state.req_context) + finished_req_ids.append(req_id) + self.primary_tier.reset_cache() - self._request_level_tiers.clear() + for req_id in finished_req_ids: + del self._req_state[req_id] self._processed_jobs_this_step = False @override