[Bugfix][KV Offload] Defer request-level cascade of in-flight primary keys (#53329)

Signed-off-by: almogtavor <almogtavor@gmail.com>
This commit is contained in:
Almog Tavor
2026-08-25 23:20:03 +00:00
committed by GitHub
parent 80771bbbdd
commit bc39ded3c4
2 changed files with 181 additions and 8 deletions
@@ -247,6 +247,35 @@ class TestExampleSecondaryTierManager:
assert tier.lookup(blocks[2], _CTX) is LookupResult.MISS
# What a request-level cascade does with a key already present in the primary
# tier, keyed by what primary lookup says about it. SUPPLY reaches the peer now,
# DEFER is retried next step, DROP is never offered again, permanently so since
# prepare_store counts the key as stored and the scheduler advances past its
# chunk. A new LookupResult has to declare which it is, and the test named
# beside it is where that behavior is actually exercised.
_CASCADE_SUPPLY_BEHAVIOR = {
# test_cascade_defers_keys_whose_primary_write_is_in_flight, after the
# writer's complete_store lands.
LookupResult.HIT: "supply",
# The same test, before it lands.
LookupResult.HIT_PENDING: "defer",
# test_cascade_drops_deferred_keys_whose_primary_write_failed, where the
# failed write frees the block.
LookupResult.MISS: "drop",
# test_cascade_rejects_retry_from_primary. The primary tier resolves every
# key it holds, so RETRY cannot reach the cascade, and parking on it would
# have no guarantee of draining, which is what makes parking HIT_PENDING
# safe. It is the one case no real primary tier can produce.
LookupResult.RETRY: "unreachable",
}
def test_every_lookup_result_has_a_declared_cascade_behavior():
"""A new LookupResult must state what the cascade does with it, rather
than falling into whichever branch happens to catch it."""
assert set(_CASCADE_SUPPLY_BEHAVIOR) == set(LookupResult)
class TestTieringOffloadingManager:
"""Tests for TieringOffloadingManager."""
@@ -1079,6 +1108,111 @@ class TestTieringOffloadingManager:
# tier2 (block-level) does not get existing blocks here.
self.secondary_tier2.submit_store.assert_not_called()
def _make_request_level_request(self, req_id: str) -> ReqContext:
"""Start a request for which tier1 asks for request-level offloading,
with tier1's submit_store wrapped so the cascade can be observed."""
self.secondary_tier1.on_new_request = lambda req_context: (
RequestOffloadingContext(policy=OffloadPolicy.REQUEST_LEVEL)
)
ctx = ReqContext(req_id=req_id)
self.manager.on_new_request(ctx)
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
return ctx
def _cascaded_keys_for(self, req_id: str) -> list[set[OffloadKey]]:
"""Key sets tier1 was asked to store on behalf of `req_id`."""
return [
set(call.args[0].keys)
for call in self.secondary_tier1.submit_store.call_args_list
if call.args[0].req_context.req_id == req_id
]
def test_cascade_rejects_retry_from_primary(self, manager_setup):
"""RETRY would be parked with no guarantee of ever draining, so the
cascade refuses it. No real primary tier returns it, so this is the one
disposition that has to be forced."""
keys = to_keys(range(3))
self._start_request()
assert self.manager.prepare_store(keys, _CTX) is not None
self.manager.complete_store(keys, _CTX, success=True)
self._simulate_on_schedule_end()
ctx = self._make_request_level_request("req_cascade")
self.manager.primary_tier.lookup = lambda key, req_context: (LookupResult.RETRY)
with pytest.raises(AssertionError):
self.manager._cascade_existing_blocks_to_request_level_tiers(keys, ctx, {0})
def _start_in_flight_primary_write(self, keys: list[OffloadKey]) -> ReqContext:
"""Leave a GPU->primary write for `keys` open, so they look present to
prepare_store but are not yet readable."""
writer_ctx = ReqContext(req_id="req_writer")
self._start_request(writer_ctx)
assert self.manager.prepare_store(keys, writer_ctx) is not None
return writer_ctx
def test_cascade_defers_keys_whose_primary_write_is_in_flight(self, manager_setup):
"""A key another request is still writing must reach the peer once the
write lands, not be dropped: prepare_store already counts it as stored
and the scheduler advances past its chunk, so nothing re-offers it."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
ctx = self._make_request_level_request("req_cascade")
result = self.manager.prepare_store(keys, ctx)
assert result is not None
assert not result.keys_to_store
assert not self._cascaded_keys_for(ctx.req_id)
self.manager.complete_store(keys, writer_ctx, success=True)
self._simulate_on_schedule_end()
assert self._cascaded_keys_for(ctx.req_id) == [set(keys)]
def test_deferred_cascade_holds_request_from_finalization(self, manager_setup):
"""A request that finishes with keys still deferred must stay alive, or
its tiers are torn down before the peer is ever served."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
ctx = self._make_request_level_request("req_cascade")
result = self.manager.prepare_store(keys, ctx)
assert result is not None
assert not result.keys_to_store
self.manager.on_request_finished(ctx)
assert ctx.req_id in self.manager._req_state
assert self.manager.has_pending_work()
self.manager.complete_store(keys, writer_ctx, success=True)
self._simulate_on_schedule_end()
assert self._cascaded_keys_for(ctx.req_id) == [set(keys)]
assert ctx.req_id not in self.manager._req_state
def test_cascade_drops_deferred_keys_whose_primary_write_failed(
self, manager_setup
):
"""A failed write frees the block, so the deferred key resolves to MISS
and the request finalizes instead of parking forever."""
keys = to_keys(range(3))
writer_ctx = self._start_in_flight_primary_write(keys)
ctx = self._make_request_level_request("req_cascade")
result = self.manager.prepare_store(keys, ctx)
assert result is not None
assert not result.keys_to_store
self.manager.on_request_finished(ctx)
self.manager.complete_store(keys, writer_ctx, success=False)
self._simulate_on_schedule_end()
assert not self._cascaded_keys_for(ctx.req_id)
assert ctx.req_id not in self.manager._req_state
assert not self.manager.has_pending_work()
def test_reset_cache_clears_orchestrator_state(self, manager_setup):
"""reset_cache wipes every kind of orchestrator state and resets
primary tier; pending submissions are dropped without being sent