[KV Offload] Add per-request offloading policy via on_new_request lifecycle hook (#43205)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Or Ozeri
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7e53283b1c
commit
a3ed5ab10c
@@ -24,7 +24,9 @@ from vllm.v1.kv_cache_interface import (
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadingEvent,
|
||||
OffloadingManager,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
get_offload_block_hash,
|
||||
)
|
||||
from vllm.v1.request import RequestStatus
|
||||
@@ -748,12 +750,8 @@ class TestSlidingWindowLookup:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bool):
|
||||
"""With do_remote_decode=True, after loading prefix blocks from CPU,
|
||||
all blocks must be re-stored — not just the newly computed ones.
|
||||
|
||||
This supports P/D disaggregation where the prefill instance offloads the
|
||||
complete KV cache so a remote decode node can consume it."""
|
||||
def test_request_level_policy_stores_all_blocks(request_runner, async_scheduling: bool):
|
||||
"""With REQUEST_LEVEL policy, all blocks are stored — including prefix hits."""
|
||||
gpu_block_size = 4
|
||||
block_size_factor = 3
|
||||
offloaded_block_size = gpu_block_size * block_size_factor
|
||||
@@ -780,12 +778,13 @@ def test_do_remote_decode_stores_all_blocks(request_runner, async_scheduling: bo
|
||||
# Reset GPU prefix cache so the next request must load from CPU.
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
|
||||
# New request with do_remote_decode=True and 2 offloaded blocks.
|
||||
# The first offloaded block matches what we stored in CPU.
|
||||
runner.new_request(
|
||||
token_ids=[0] * offloaded_block_size * 2,
|
||||
kv_transfer_params={"do_remote_decode": True},
|
||||
# Manager returns REQUEST_LEVEL for the next request.
|
||||
runner.manager.on_new_request.return_value = RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
|
||||
# New request with 2 offloaded blocks; first matches what's in CPU.
|
||||
runner.new_request(token_ids=[0] * offloaded_block_size * 2)
|
||||
runner.connector_scheduler._maximal_prefix_lookup = lambda key, req_context: 1
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
@@ -957,9 +956,10 @@ def test_max_offload_tokens_validation(request_runner, async_scheduling: bool):
|
||||
)
|
||||
|
||||
def setup(r, max_offload_tokens):
|
||||
r.new_request(token_ids=[0] * offloaded_block_size * 3)
|
||||
req = r.scheduler.requests[str(r.req_id)]
|
||||
req.kv_transfer_params = {"max_offload_tokens": max_offload_tokens}
|
||||
r.new_request(
|
||||
token_ids=[0] * offloaded_block_size * 3,
|
||||
kv_transfer_params={"max_offload_tokens": max_offload_tokens},
|
||||
)
|
||||
r.manager.prepare_store.side_effect = (
|
||||
lambda keys, req_context: generate_store_output(keys)
|
||||
)
|
||||
|
||||
@@ -43,6 +43,7 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingSpec,
|
||||
OffloadKey,
|
||||
PrepareStoreOutput,
|
||||
RequestOffloadingContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.worker.worker import (
|
||||
@@ -119,6 +120,7 @@ class MockOffloadingSpec(OffloadingSpec):
|
||||
self.manager.lookup.return_value = 0
|
||||
self.manager.prepare_load = lambda keys, req_context: MockLoadStoreSpec(keys)
|
||||
self.manager.lookup.return_value = False
|
||||
self.manager.on_new_request.return_value = RequestOffloadingContext()
|
||||
self.handler = MockOffloadingHandler()
|
||||
|
||||
def get_manager(self) -> OffloadingManager:
|
||||
|
||||
@@ -19,7 +19,9 @@ import torch
|
||||
|
||||
from vllm.v1.kv_offload.base import (
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
make_offload_key,
|
||||
)
|
||||
from vllm.v1.kv_offload.tiering.example.manager import ExampleSecondaryTierManager
|
||||
@@ -412,6 +414,78 @@ class TestTieringOffloadingManager:
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert job_metadata.req_context is ctx
|
||||
|
||||
def test_on_new_request_lifecycle(self, manager_setup):
|
||||
"""Policy defaults to BLOCK_LEVEL, escalates when a tier requests it,
|
||||
and is cleaned up on on_request_finished."""
|
||||
# Default: all tiers return BLOCK_LEVEL
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.BLOCK_LEVEL
|
||||
self.manager.on_request_finished(ctx)
|
||||
|
||||
# Escalate: tier1 requests REQUEST_LEVEL
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_policy_lifecycle_2")
|
||||
result = self.manager.on_new_request(ctx)
|
||||
assert result.policy == OffloadPolicy.REQUEST_LEVEL
|
||||
assert ctx.req_id in self.manager._request_level_tiers
|
||||
|
||||
# Cleanup
|
||||
self.manager.on_request_finished(ctx)
|
||||
assert ctx.req_id not in self.manager._request_level_tiers
|
||||
|
||||
def test_prepare_store_cascades_existing_blocks_to_request_level_tiers(
|
||||
self, manager_setup
|
||||
):
|
||||
"""prepare_store cascades hit blocks to request-level tiers only."""
|
||||
# Store some blocks to primary first
|
||||
existing_blocks = to_keys(range(3))
|
||||
result = self.manager.prepare_store(existing_blocks, _CTX)
|
||||
assert result is not None
|
||||
self.manager.complete_store(existing_blocks, _CTX, success=True)
|
||||
# Drain cascade completions
|
||||
list(self.manager.take_events())
|
||||
list(self.manager.take_events())
|
||||
|
||||
# Make tier1 request-level, tier2 stays block-level
|
||||
self.secondary_tier1.on_new_request = (
|
||||
lambda req_context: RequestOffloadingContext(
|
||||
policy=OffloadPolicy.REQUEST_LEVEL
|
||||
)
|
||||
)
|
||||
|
||||
ctx = ReqContext(req_id="req_cascade")
|
||||
self.manager.on_new_request(ctx)
|
||||
|
||||
# Spy on submit_store
|
||||
self.secondary_tier1.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier1.submit_store
|
||||
)
|
||||
self.secondary_tier2.submit_store = MagicMock(
|
||||
wraps=self.secondary_tier2.submit_store
|
||||
)
|
||||
|
||||
# Call prepare_store with existing + new blocks
|
||||
new_blocks = to_keys(range(3, 5))
|
||||
all_blocks = existing_blocks + new_blocks
|
||||
result = self.manager.prepare_store(all_blocks, ctx)
|
||||
assert result is not None
|
||||
assert set(result.keys_to_store) == set(new_blocks)
|
||||
|
||||
# Only tier1 (request-level) should get existing blocks cascaded now.
|
||||
# New blocks are cascaded to ALL tiers later via complete_store().
|
||||
self.secondary_tier1.submit_store.assert_called_once()
|
||||
job_metadata = self.secondary_tier1.submit_store.call_args.args[0]
|
||||
assert set(job_metadata.keys) == set(existing_blocks)
|
||||
|
||||
# tier2 (block-level) does not get existing blocks here.
|
||||
self.secondary_tier2.submit_store.assert_not_called()
|
||||
|
||||
|
||||
class TestTieringOffloadingWithoutSecondaryTiers:
|
||||
"""Test TieringOffloadingManager with no secondary tiers (backward compat)."""
|
||||
|
||||
Reference in New Issue
Block a user