[KV Offload] Add per-request offloading policy via on_new_request lifecycle hook (#43205)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ronen Schaffer
2026-05-28 20:45:18 +00:00
committed by GitHub
co-authored by Or Ozeri mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7e53283b1c
commit a3ed5ab10c
11 changed files with 294 additions and 45 deletions
@@ -24,7 +24,9 @@ from vllm.v1.kv_cache_interface import (
from vllm.v1.kv_offload.base import (
OffloadingEvent,
OffloadingManager,
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
get_offload_block_hash,
)
from vllm.v1.request import RequestStatus
@@ -748,12 +750,8 @@ class TestSlidingWindowLookup:
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bool):
"""With do_remote_decode=True, after loading prefix blocks from CPU,
all blocks must be re-stored — not just the newly computed ones.
This supports P/D disaggregation where the prefill instance offloads the
complete KV cache so a remote decode node can consume it."""
def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool):
"""With REQUEST_LEVEL policy, all blocks are stored — including prefix hits."""
gpu_block_size = 4
block_size_factor = 3
offloaded_block_size = gpu_block_size * block_size_factor
@@ -780,12 +778,13 @@ def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bo
# Reset GPU prefix cache so the next request must load from CPU.
runner.scheduler.reset_prefix_cache()
# New request with do_remote_decode=True and 2 offloaded blocks.
# The first offloaded block matches what we stored in CPU.
runner.new_request(
token_ids=[0] * offloaded_block_size * 2,
kv_transfer_params={"do_remote_decode": True},
# Manager returns REQUEST_LEVEL for the next request.
runner.manager.on_new_request.return_value = RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
# New request with 2 offloaded blocks; first matches what's in CPU.
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
@@ -957,9 +956,10 @@ def test_max_offload_tokens_validation(request_runner, async_scheduling: bool):
)
def setup(r, max_offload_tokens):
r.new_request(token_ids=[0] * offloaded_block_size * 3)
req = r.scheduler.requests[str(r.req_id)]
req.kv_transfer_params = {"max_offload_tokens": max_offload_tokens}
r.new_request(
token_ids=[0] * offloaded_block_size * 3,
kv_transfer_params={"max_offload_tokens": max_offload_tokens},
)
r.manager.prepare_store.side_effect = (
lambda keys, req_context: generate_store_output(keys)
)
@@ -43,6 +43,7 @@ from vllm.v1.kv_offload.base import (
OffloadingSpec,
OffloadKey,
PrepareStoreOutput,
RequestOffloadingContext,
make_offload_key,
)
from vllm.v1.kv_offload.worker.worker import (
@@ -119,6 +120,7 @@ class MockOffloadingSpec(OffloadingSpec):
self.manager.lookup.return_value = 0
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
self.manager.lookup.return_value = False
self.manager.on_new_request.return_value = RequestOffloadingContext()
self.handler = MockOffloadingHandler()
def get_manager(self) -> OffloadingManager:
@@ -19,7 +19,9 @@ import torch
from vllm.v1.kv_offload.base import (
OffloadKey,
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
make_offload_key,
)
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
@@ -412,6 +414,78 @@ class TestTieringOffloadingManager:
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert job_metadata.req_context is ctx
def test_on_new_request_lifecycle(self, manager_setup):
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
and is cleaned up on on_request_finished."""
# Default: all tiers return BLOCK_LEVEL
ctx = ReqContext(req_id="req_policy_lifecycle")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.BLOCK_LEVEL
self.manager.on_request_finished(ctx)
# Escalate: tier1 requests REQUEST_LEVEL
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
)
ctx = ReqContext(req_id="req_policy_lifecycle_2")
result = self.manager.on_new_request(ctx)
assert result.policy == OffloadPolicy.REQUEST_LEVEL
assert ctx.req_id in self.manager._request_level_tiers
# Cleanup
self.manager.on_request_finished(ctx)
assert ctx.req_id not in self.manager._request_level_tiers
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
self, manager_setup
):
"""prepare_store cascades hit blocks to request-level tiers only."""
# Store some blocks to primary first
existing_blocks = to_keys(range(3))
result = self.manager.prepare_store(existing_blocks, _CTX)
assert result is not None
self.manager.complete_store(existing_blocks, _CTX, success=True)
# Drain cascade completions
list(self.manager.take_events())
list(self.manager.take_events())
# Make tier1 request-level, tier2 stays block-level
self.secondary_tier1.on_new_request = (
lambda req_context: RequestOffloadingContext(
policy=OffloadPolicy.REQUEST_LEVEL
)
)
ctx = ReqContext(req_id="req_cascade")
self.manager.on_new_request(ctx)
# Spy on submit_store
self.secondary_tier1.submit_store = MagicMock(
wraps=self.secondary_tier1.submit_store
)
self.secondary_tier2.submit_store = MagicMock(
wraps=self.secondary_tier2.submit_store
)
# Call prepare_store with existing + new blocks
new_blocks = to_keys(range(3, 5))
all_blocks = existing_blocks + new_blocks
result = self.manager.prepare_store(all_blocks, ctx)
assert result is not None
assert set(result.keys_to_store) == set(new_blocks)
# Only tier1 (request-level) should get existing blocks cascaded now.
# New blocks are cascaded to ALL tiers later via complete_store().
self.secondary_tier1.submit_store.assert_called_once()
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
assert set(job_metadata.keys) == set(existing_blocks)
# tier2 (block-level) does not get existing blocks here.
self.secondary_tier2.submit_store.assert_not_called()
class TestTieringOffloadingWithoutSecondaryTiers:
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""