Fix KV offload request-finished lifecycle contract (#46284)

Signed-off-by: test test <2260891073@qq.com>
Signed-off-by: Rui Yin <2260891073@qq.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Rui Yin
2026-06-24 15:42:40 +03:00
committed by GitHub
co-authored by Or Ozeri
parent d4448b511d
commit cf9fd6457e
5 changed files with 334 additions and 112 deletions
@@ -216,17 +216,14 @@ def test_request_preemption(request_runner, async_scheduling: bool):
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_no_offload_call_after_on_request_finished(
def test_on_request_finished_is_not_deferred_until_store_completion(
request_runner, async_scheduling: bool
):
"""on_request_finished is not issued before a per-request offload
call.
"""on_request_finished fires when no more stores will be submitted.
A request can finish while its GPU->primary store is still in flight; the
later worker completion then drives complete_store. The scheduler defers
on_request_finished until the request is finished AND has no in-flight
transfer jobs, so complete_store is observed BEFORE on_request_finished,
and it is called exactly once.
A request can finish while its GPU->primary store is still in flight. The
manager-level hook should not wait for that completion; complete_store may
still arrive afterward for already-submitted transfer jobs.
"""
block_size = 4
block_size_factor = 3
@@ -263,27 +260,34 @@ def test_no_offload_call_after_on_request_finished(
complete_transfers=False,
)
# Finish the request, completing its pending stores. on_request_finished is
# deferred until the stores drain, so it lands after the last complete_store.
# 4 offloaded blocks are stored (2 prompt + 2 decode) -> 4 * block_size_factor
# GPU blocks.
# Finish the request while its stores are still in flight. The hook should
# fire immediately even though no complete_store has arrived yet.
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=tuple(range(4 * block_size_factor)),
complete_transfers=False,
)
req_id = str(runner.req_id)
assert calls == [("on_request_finished", req_id)], calls
# Drain the stores afterward. The already-submitted complete_store calls
# are allowed to arrive after on_request_finished.
runner.run(
decoded_tokens=[],
complete_transfers=True,
expected_stored=tuple(range(4 * block_size_factor)),
)
# on_request_finished is issued exactly once.
assert calls.count(("on_request_finished", req_id)) == 1, calls
finished_idx = calls.index(("on_request_finished", req_id))
store_indices = [i for i, c in enumerate(calls) if c == ("complete_store", req_id)]
# All of the request's complete_store calls must precede its single
# on_request_finished.
# The request-level hook no longer waits for already-submitted transfers.
assert store_indices, calls
assert max(store_indices) < finished_idx, calls
assert finished_idx < min(store_indices), calls
@pytest.mark.parametrize("async_scheduling", [True, False])
@@ -1232,11 +1236,8 @@ def test_reset_cache(request_runner, async_scheduling: bool):
def test_reset_cache_finalizes_finished_request_with_pending_store(
request_runner, async_scheduling: bool
):
"""reset_cache must finalize a finished request whose in-flight stores it
discards: call on_request_finished and drop its _req_status entry.
Otherwise the deferred hook (which waits for the now-discarded jobs to
complete) never fires and the entry leaks.
"""reset_cache drops a finished request whose in-flight stores it discards
without calling on_request_finished twice.
"""
block_size = 4
block_size_factor = 3
@@ -1273,14 +1274,15 @@ def test_reset_cache_finalizes_finished_request_with_pending_store(
assert any(job.is_store for job in cs._jobs.values())
# Finish the request while its store is still in flight. request_finished
# takes the defer branch (pending jobs), so on_request_finished is NOT
# called yet and the entry stays tracked.
# fires the hook eagerly, but the entry stays tracked so later completions
# can still call complete_store().
req_status.req.status = RequestStatus.FINISHED_STOPPED
cs.request_finished(req_status.req)
assert finalized == []
assert finalized == [req_id]
assert req_id in cs._req_status
# reset_cache discards the in-flight store; it must finalize the request.
# reset_cache discards the in-flight store and drops the state without a
# duplicate on_request_finished call.
cs.reset_cache()
assert finalized == [req_id]
assert req_id not in cs._req_status
@@ -235,11 +235,16 @@ class TestTieringOffloadingManager:
self.manager.on_schedule_end()
list(self.manager.take_events())
def _start_request(self, req_context: ReqContext = _CTX):
if req_context.req_id not in self.manager._req_state:
self.manager.on_new_request(req_context)
def test_basic_store_to_primary(self, manager_setup):
"""Test basic store operation to primary tier."""
blocks = to_keys(range(3))
# Prepare store
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 3
@@ -262,6 +267,7 @@ class TestTieringOffloadingManager:
)
# Store to primary
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
@@ -285,6 +291,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(3))
# Store to primary
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
self.manager.complete_store(blocks, _CTX, success=True)
@@ -330,6 +337,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(3))
# Store blocks
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
@@ -366,6 +374,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(5))
# Store first 3 blocks to primary
self._start_request()
self.manager.prepare_store(blocks[:3], _CTX)
self.manager.complete_store(blocks[:3], _CTX, success=True)
@@ -377,6 +386,7 @@ class TestTieringOffloadingManager:
# Primary tier has capacity of 5 blocks
# First, fill the primary tier
blocks = to_keys(range(5))
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
assert len(result.keys_to_store) == 5
@@ -399,6 +409,7 @@ class TestTieringOffloadingManager:
blocks = to_keys(range(3))
# Store blocks
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
self._simulate_on_schedule_end()
@@ -431,6 +442,7 @@ class TestTieringOffloadingManager:
)
# Prepare store
self._start_request()
result = self.manager.prepare_store(blocks, _CTX)
assert result is not None
@@ -521,6 +533,7 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_ctx", kv_transfer_params={"key": "value"})
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.complete_store(blocks, ctx, success=True)
@@ -528,6 +541,172 @@ class TestTieringOffloadingManager:
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx
def test_on_request_finished_delays_secondary_until_store_submitted(
self, manager_setup
):
"""Manager hook is eager; secondary hooks wait for cascade submission."""
blocks = to_keys(range(2))
ctx = ReqContext(req_id="req_delayed_secondary")
calls: list[tuple[str, str]] = []
self.primary_tier.on_request_finished = MagicMock(
side_effect=lambda req_context: calls.append(
("primary_finish", req_context.req_id)
)
)
original_submit_store1 = self.secondary_tier1.submit_store
original_submit_store2 = self.secondary_tier2.submit_store
def submit_store1(job_metadata):
calls.append(("submit_store_1", job_metadata.req_context.req_id))
return original_submit_store1(job_metadata)
def submit_store2(job_metadata):
calls.append(("submit_store_2", job_metadata.req_context.req_id))
return original_submit_store2(job_metadata)
self.secondary_tier1.submit_store = MagicMock(side_effect=submit_store1)
self.secondary_tier2.submit_store = MagicMock(side_effect=submit_store2)
self.secondary_tier1.on_request_finished = MagicMock(
side_effect=lambda req_context: calls.append(
("secondary_finish_1", req_context.req_id)
)
)
self.secondary_tier2.on_request_finished = MagicMock(
side_effect=lambda req_context: calls.append(
("secondary_finish_2", req_context.req_id)
)
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.on_request_finished(ctx)
assert calls == [("primary_finish", ctx.req_id)]
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=True)
assert calls == [
("primary_finish", ctx.req_id),
("submit_store_1", ctx.req_id),
("submit_store_2", ctx.req_id),
("secondary_finish_1", ctx.req_id),
("secondary_finish_2", ctx.req_id),
]
def test_failed_store_finalizes_finished_request(self, manager_setup):
"""Failed primary stores still unblock secondary finalization."""
blocks = to_keys(range(2))
ctx = ReqContext(req_id="req_failed_store_finalize")
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.complete_store(blocks, ctx, success=False)
self.secondary_tier1.submit_store.assert_not_called()
self.secondary_tier2.submit_store.assert_not_called()
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert ctx.req_id not in self.manager._req_state
def test_zero_store_request_finalizes_immediately(self, manager_setup):
"""Requests with no pending stores finalize secondary tiers immediately."""
ctx = ReqContext(req_id="req_zero_store_finalize")
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert ctx.req_id not in self.manager._req_state
def test_reset_cache_finalizes_delayed_secondary_request(self, manager_setup):
"""reset_cache abandons pending primary stores and finalizes secondaries."""
blocks = to_keys(range(2))
ctx = ReqContext(req_id="req_reset_finalize_secondary")
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.prepare_store(blocks, ctx)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.reset_cache()
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert self.manager._req_state == {}
def test_reset_cache_clears_pending_primary_stores_for_active_request(
self, manager_setup
):
"""reset_cache drops active pending stores so resumed requests finalize."""
initial_blocks = to_keys(range(2))
resumed_blocks = to_keys(range(2, 4))
ctx = ReqContext(req_id="req_reset_resume")
self.secondary_tier1.on_request_finished = MagicMock(
wraps=self.secondary_tier1.on_request_finished
)
self.secondary_tier2.on_request_finished = MagicMock(
wraps=self.secondary_tier2.on_request_finished
)
self._start_request(ctx)
self.manager.prepare_store(initial_blocks, ctx)
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 1
self.manager.reset_cache()
assert ctx.req_id in self.manager._req_state
assert self.manager._req_state[ctx.req_id].pending_primary_stores == 0
self.secondary_tier1.on_request_finished.assert_not_called()
self.secondary_tier2.on_request_finished.assert_not_called()
self.manager.prepare_store(resumed_blocks, ctx)
self.manager.complete_store(resumed_blocks, ctx, success=True)
self.manager.on_request_finished(ctx)
self.secondary_tier1.on_request_finished.assert_called_once_with(ctx)
self.secondary_tier2.on_request_finished.assert_called_once_with(ctx)
assert ctx.req_id not in self.manager._req_state
def test_on_new_request_lifecycle(self, manager_setup):
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
and is cleaned up on on_request_finished."""
@@ -535,23 +714,25 @@ class TestTieringOffloadingManager:
ctx = ReqContext(req_id="req_policy_lifecycle")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.BLOCK_LEVEL
assert self.manager._req_state[ctx.req_id].request_level_tiers is None
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._req_state
# Escalate: tier1 requests REQUEST_LEVEL
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
ctx = ReqContext(req_id="req_policy_lifecycle_2")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.REQUEST_LEVEL
assert ctx.req_id in self.manager._request_level_tiers
assert self.manager._req_state[ctx.req_id].request_level_tiers == {
self.secondary_tier1
}
# Cleanup
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._request_level_tiers
assert ctx.req_id not in self.manager._req_state
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
self, manager_setup
@@ -559,6 +740,7 @@ class TestTieringOffloadingManager:
"""prepare_store cascades hit blocks to request-level tiers only."""
# Store some blocks to primary first
existing_blocks = to_keys(range(3))
self._start_request()
result = self.manager.prepare_store(existing_blocks, _CTX)
assert result is not None
self.manager.complete_store(existing_blocks, _CTX, success=True)
@@ -566,10 +748,8 @@ class TestTieringOffloadingManager:
self._simulate_on_schedule_end()
# Make tier1 request-level, tier2 stays block-level
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
ctx = ReqContext(req_id="req_cascade")
@@ -599,14 +779,15 @@ class TestTieringOffloadingManager:
# tier2 (block-level) does not get existing blocks here.
self.secondary_tier2.submit_store.assert_not_called()
def test_reset_cache_clears_all_state(self, manager_setup):
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
"""reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent
to the secondary tier."""
to the secondary tier. Active request state is retained."""
# Cascade — populates primary blocks and leaves cascade jobs
# in _transfer_jobs (the synchronous example tier has already
# queued completions); reset_cache's drain loop will pick them up.
blocks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
@@ -619,13 +800,14 @@ class TestTieringOffloadingManager:
assert self.manager._pending_load_submissions
# Request-level tier registration.
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
self.manager.on_new_request(ReqContext(req_id="rl"))
assert self.manager._request_level_tiers
rl_ctx = ReqContext(req_id="rl")
self.manager.on_new_request(rl_ctx)
assert self.manager._req_state[rl_ctx.req_id].request_level_tiers == {
self.secondary_tier1
}
# Mark this step as already polled (reset_cache must clear it).
self.manager._processed_jobs_this_step = True
@@ -640,7 +822,7 @@ class TestTieringOffloadingManager:
# Orchestrator state cleared.
assert self.manager._transfer_jobs == {}
assert self.manager._pending_load_submissions == {}
assert self.manager._request_level_tiers == {}
assert set(self.manager._req_state) == {_CTX.req_id, rl_ctx.req_id}
assert self.manager._processed_jobs_this_step is False
# Primary tier reset to a fresh state.
@@ -668,6 +850,7 @@ class TestTieringOffloadingManager:
# Drive a cascade so a job lands in _transfer_jobs.
blocks = to_keys(range(3))
self._start_request()
self.manager.prepare_store(blocks, _CTX)
self.manager.complete_store(blocks, _CTX, success=True)
assert self.manager._transfer_jobs
@@ -696,6 +879,7 @@ class TestTieringOffloadingWithoutSecondaryTiers:
blocks = to_keys(range(3))
# Should work like a regular OffloadingManager
manager.on_new_request(_CTX)
result = manager.prepare_store(blocks, _CTX)
assert result is not None
manager.complete_store(blocks, _CTX, success=True)
@@ -1140,11 +1140,6 @@ class OffloadingConnectorScheduler:
del self._jobs[job_id]
req_status.transfer_jobs.remove(job_id)
if not req_status.transfer_jobs and req_status.req.is_finished():
# Deferred from request_finished: the request's last in-flight
# job is now done, so fire the finalize hook here, after the
# final complete_store/complete_load above (and any submit_store
# the complete_store cascade issued).
self.manager.on_request_finished(req_status.req_context)
del self._req_status[job_status.req_id]
def get_stats(self) -> OffloadingConnectorStats | None:
@@ -1181,20 +1176,23 @@ class OffloadingConnectorScheduler:
if req_status is None:
# Untracked request (offloading never started): no in-flight jobs,
# nothing was deferred, so finalize immediately.
self.manager.on_request_finished(_create_req_context(request))
req_context = _create_req_context(request)
self.manager.on_new_request(req_context)
self.manager.on_request_finished(req_context)
return False, None
self.manager.on_request_finished(req_status.req_context)
if not req_status.transfer_jobs:
# No in-flight jobs: all per-request calls are done, finalize now.
self.manager.on_request_finished(req_status.req_context)
# No in-flight jobs: no later complete_store()/complete_load() calls
# need this request's state.
del self._req_status[request.request_id]
return False, None
# In-flight jobs remain, so defer on_request_finished to
# update_connector_output, which fires it once the last job completes
# (after the final complete_store and any cascade submit_store it
# issues). These pending stores outlive the request's block ownership;
# register them so future reuse of those blocks triggers a flush.
# In-flight jobs remain after the request stopped. Their completion may
# still call manager.complete_store()/complete_load(), so keep req_status.
# Pending stores outlive the request's block ownership; register them so
# future reuse of those blocks triggers a flush.
for job_id in req_status.transfer_jobs:
job_status = self._jobs[job_id]
for bid in job_status.non_sliding_window_block_ids or ():
@@ -1225,14 +1223,8 @@ class OffloadingConnectorScheduler:
# Flush all in-flight jobs
self._current_batch_jobs_to_flush.update(self._jobs.keys())
# A finished request may still be tracked here with in-flight jobs that
# this reset discards, so its deferred on_request_finished() would never
# fire (completions are skipped as stale) and its _req_status entry would
# leak. Finalize such requests now, before resetting the manager.
# list() snapshots because we delete while iterating.
for req_id, status in list(self._req_status.items()):
if status.req.is_finished():
self.manager.on_request_finished(status.req_context)
del self._req_status[req_id]
# Reset offloading manager cache
+9 -9
View File
@@ -276,16 +276,16 @@ class OffloadingManager(ABC):
"""
Called when a request has finished.
By the time this is called, all per-request offload calls for this
request (prepare_store/complete_store, prepare_load/complete_load,
touch, lookup) have already been issued, and none will follow. The
scheduler defers this call until the request is finished and has no
in-flight transfer jobs.
By the time this is called, the scheduler will issue no more
submit-side calls for this request, such as prepare_store() and
prepare_load(). Completion callbacks for already-submitted transfers
(complete_store() and complete_load()) may still arrive afterward.
Note this signals only that no further calls will be made; it does NOT
imply the data has been persisted. Asynchronous transfers already
submitted for this request (e.g. CPU->secondary cascades) may still be
in flight. This is the right place to release per-request bookkeeping.
This hook does NOT imply the data has been persisted. Asynchronous
transfers already submitted for this request may still be in flight.
Managers that cascade to lower tiers should delay those tiers'
on_request_finished() calls until no more lower-tier submit calls can
be issued for this request.
Args:
req_context: per-request context.
+84 -40
View File
@@ -20,7 +20,6 @@ Key Design Principles:
protecting blocks from eviction until complete_read() is called
"""
from collections import defaultdict
from collections.abc import Collection, Iterable, Sequence
from dataclasses import dataclass, field
@@ -62,6 +61,14 @@ class PendingPromotion:
block_ids: list[int] = field(default_factory=list)
@dataclass(slots=True)
class RequestState:
req_context: ReqContext
pending_primary_stores: int = 0
is_finished: bool = False
request_level_tiers: set[SecondaryTierManager] | None = None
class CPUPrimaryTierOffloadingManager(CPUOffloadingManager):
"""CPUOffloadingManager with a primary/secondary transfer interface.
@@ -166,12 +173,10 @@ class TieringOffloadingManager(OffloadingManager):
# Reset at the end of each step in on_schedule_end().
self._processed_jobs_this_step: bool = False
# Per-request set of secondary tiers that requested REQUEST_LEVEL
# policy. Populated in on_new_request(),
# cleaned up in on_request_finished().
self._request_level_tiers: defaultdict[str, set[SecondaryTierManager]] = (
defaultdict(set)
)
# Per-request state for prepared GPU->primary stores and finalization.
# Secondary tiers are finalized only after pending primary stores reach
# complete_store(), since complete_store() can still submit cascades.
self._req_state: dict[str, RequestState] = {}
def _next_job_id(self) -> JobId:
"""Generate a unique job ID for async transfer tracking."""
@@ -432,9 +437,13 @@ class TieringOffloadingManager(OffloadingManager):
if primary_result is None:
return None
if primary_result.keys_to_store:
state = self._req_state[req_context.req_id]
state.pending_primary_stores += 1
# Step 3: For request-level tiers, cascade blocks already in primary
request_level_tiers = self._request_level_tiers.get(req_context.req_id)
if request_level_tiers is not None:
request_level_tiers = self._req_state[req_context.req_id].request_level_tiers
if request_level_tiers:
keys_to_store_set = set(primary_result.keys_to_store)
keys_already_in_primary = tuple(
k for k in keys if k not in keys_to_store_set
@@ -508,36 +517,38 @@ class TieringOffloadingManager(OffloadingManager):
# Step 1: Complete store in primary tier (makes blocks loadable)
self.primary_tier.complete_store(keys, req_context, success)
if not success:
# If GPU→Primary transfer failed, don't cascade to secondary tiers
return
if success:
# Step 2: Cascade to ALL secondary tiers
# For each secondary tier, call primary.prepare_read() to get the
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier in self.secondary_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
# Step 2: Cascade to ALL secondary tiers
# For each secondary tier, call primary.prepare_read() to get the
# LoadStoreSpec AND to increment ref_cnt (protecting blocks from
# eviction during the async transfer). One prepare_read() call per
# secondary tier.
for tier in self.secondary_tiers:
primary_blocks_spec = self.primary_tier.prepare_read(keys, req_context)
# Submit async store job: primary→secondary
job_id = self._next_job_id()
# Submit async store job: primary→secondary
job_id = self._next_job_id()
# Track this store job
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
# Track this store job
assert isinstance(primary_blocks_spec, CPULoadStoreSpec)
job_metadata = JobMetadata(
job_id=job_id,
keys=keys,
block_ids=primary_blocks_spec.block_ids,
is_promotion=False,
req_context=req_context,
)
self._transfer_jobs[job_id] = job_metadata
tier.submit_store(job_metadata)
tier.submit_store(job_metadata)
# Note: The async transfers are now in flight. Their completion is
# tracked via get_finished_jobs() / _maybe_process_finished_jobs().
req_id = req_context.req_id
state = self._req_state[req_id]
assert state.pending_primary_stores > 0
state.pending_primary_stores -= 1
self._maybe_finalize_request(req_id)
@override
def on_new_request(self, req_context: ReqContext) -> RequestOffloadingContext:
@@ -547,14 +558,18 @@ class TieringOffloadingManager(OffloadingManager):
Returns REQUEST_LEVEL if ANY secondary tier wants request-level.
Only stores REQUEST_LEVEL tier decisions for use in prepare_store.
"""
state = RequestState(req_context=req_context)
for tier in self.secondary_tiers:
tier_ctx = tier.on_new_request(req_context)
if tier_ctx.policy == OffloadPolicy.REQUEST_LEVEL:
self._request_level_tiers[req_context.req_id].add(tier)
if state.request_level_tiers is None:
state.request_level_tiers = set()
state.request_level_tiers.add(tier)
self._req_state[req_context.req_id] = state
policy = (
OffloadPolicy.REQUEST_LEVEL
if req_context.req_id in self._request_level_tiers
if state.request_level_tiers
else OffloadPolicy.BLOCK_LEVEL
)
return RequestOffloadingContext(policy=policy)
@@ -562,9 +577,26 @@ class TieringOffloadingManager(OffloadingManager):
@override
def on_request_finished(self, req_context: ReqContext) -> None:
self.primary_tier.on_request_finished(req_context)
state = self._req_state[req_context.req_id]
state.is_finished = True
self._maybe_finalize_request(req_context.req_id)
def _maybe_finalize_request(self, req_id: str) -> None:
"""Finalize secondary tiers once no more store cascades can be submitted.
Finalization means forwarding on_request_finished() to secondary tiers.
It is delayed until pending GPU->primary stores finish, since their
complete_store() callbacks may still submit primary->secondary stores.
"""
state = self._req_state[req_id]
if not state.is_finished:
return
if state.pending_primary_stores != 0:
return
for tier in self.secondary_tiers:
tier.on_request_finished(req_context)
self._request_level_tiers.pop(req_context.req_id, None)
tier.on_request_finished(state.req_context)
del self._req_state[req_id]
@override
def on_schedule_end(self) -> None:
@@ -604,7 +636,7 @@ class TieringOffloadingManager(OffloadingManager):
@override
def reset_cache(self) -> None:
"""Drop all tracked state in the orchestrator and primary tier.
"""Reset transfer bookkeeping and primary-tier cache.
Called during sleep, weight update, or resume. Each secondary tier
drains its in-flight transfers via drain_jobs() so no tier I/O is
@@ -613,7 +645,9 @@ class TieringOffloadingManager(OffloadingManager):
from reusing primary slots while a transfer is mid-copy.
Secondary tiers are intentionally not reset: persistent stores
(FS, network) keep their data across resets.
(FS, network) keep their data across resets. Active request state is
retained so those requests can continue after the reset; finished
requests are finalized and removed.
"""
for tier in self.secondary_tiers:
tier.drain_jobs()
@@ -626,9 +660,19 @@ class TieringOffloadingManager(OffloadingManager):
# called so no tier I/O is touching that memory.
self._pending_load_submissions.clear()
finished_req_ids = []
for req_id, state in self._req_state.items():
state.pending_primary_stores = 0
if not state.is_finished:
continue
for tier in self.secondary_tiers:
tier.on_request_finished(state.req_context)
finished_req_ids.append(req_id)
self.primary_tier.reset_cache()
self._request_level_tiers.clear()
for req_id in finished_req_ids:
del self._req_state[req_id]
self._processed_jobs_this_step = False
@override