[KV Offload] Add per-request offloading policy via on_new_request lifecycle hook (#43205)

Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com>
Co-authored-by: Or Ozeri <or@ozery.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Ronen Schaffer
2026-05-28 20:45:18 +00:00
committed by GitHub
co-authored by Or Ozeri mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 7e53283b1c
commit a3ed5ab10c
11 changed files with 294 additions and 45 deletions
@@ -29,7 +29,9 @@ from vllm.v1.kv_offload.base import (
OffloadingManager,
OffloadingSpec,
OffloadKey,
OffloadPolicy,
ReqContext,
RequestOffloadingContext,
get_offload_block_hash,
make_offload_key,
)
@@ -172,8 +174,9 @@ class RequestGroupState:
class RequestOffloadState:
config: SchedulerOffloadConfig
req: Request
req_context: ReqContext
offloading_context: RequestOffloadingContext
group_states: tuple[RequestGroupState, ...] = field(init=False)
req_context: ReqContext = field(init=False)
# upper bound on tokens to offload for this request; None means no cap
max_offload_tokens: int | None = None
# number of hits in the GPU cache
@@ -186,10 +189,6 @@ class RequestOffloadState:
self.group_states = tuple(
RequestGroupState() for _ in self.config.kv_group_configs
)
self.req_context = ReqContext(
req_id=self.req.request_id,
kv_transfer_params=self.req.kv_transfer_params,
)
params = self.req.kv_transfer_params
# NOTE: This field is experimental and subject to change in the future.
@@ -248,6 +247,13 @@ class RequestOffloadState:
)
def _create_req_context(req: Request) -> ReqContext:
return ReqContext(
req_id=req.request_id,
kv_transfer_params=req.kv_transfer_params,
)
class OffloadingConnectorScheduler:
"""Implementation of Scheduler side methods"""
@@ -512,6 +518,18 @@ class OffloadingConnectorScheduler:
return num_hit_tokens
def on_new_request(self, request: Request) -> None:
"""Called when a new request is added to the scheduler."""
req_context = _create_req_context(request)
offloading_context = self.manager.on_new_request(req_context)
req_status = RequestOffloadState(
config=self.config,
req=request,
req_context=req_context,
offloading_context=offloading_context,
)
self._req_status[request.request_id] = req_status
def get_num_new_matched_tokens(
self, request: Request, num_computed_tokens: int
) -> tuple[int | None, bool]:
@@ -534,24 +552,15 @@ class OffloadingConnectorScheduler:
- `True` if tokens will be loaded asynchronously
(between scheduler steps).
"""
is_new_request = False
if req_status := self._req_status.get(request.request_id):
# make sure block IDs are cleared
for group_state in req_status.group_states:
group_state.block_ids.clear()
else:
is_new_request = True
req_status = RequestOffloadState(config=self.config, req=request)
self._req_status[request.request_id] = req_status
req_status = self._req_status[request.request_id]
for group_state in req_status.group_states:
group_state.block_ids.clear()
req_status.update_offload_keys()
req_status.num_locally_computed_tokens = num_computed_tokens
num_hit_tokens = self._lookup(req_status)
if is_new_request:
req_status.update_num_hit_blocks(
num_computed_tokens + (num_hit_tokens or 0)
)
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
self._touch(req_status)
@@ -568,9 +577,6 @@ class OffloadingConnectorScheduler:
num_locally_computed_tokens = req_status.num_locally_computed_tokens
num_cached_tokens = num_locally_computed_tokens + num_external_tokens
params = req_status.req_context.kv_transfer_params
do_remote_decode = params is not None and params.get("do_remote_decode")
keys_to_load: list[OffloadKey] = []
dst_block_ids: list[int] = []
# per group
@@ -624,10 +630,10 @@ class OffloadingConnectorScheduler:
group_sizes.append(num_pending_gpu_blocks)
block_indices.append(num_locally_computed_gpu_blocks)
if not do_remote_decode:
# For P/D prefill requests (do_remote_decode=True), we do
# NOT skip saving the hit prefix, as we need to stream the
# entire KV cache so a remote decode node can consume it.
# Skip prefix-hit blocks for block-level policy; for
# request-level, next_stored_block_idx stays at 0 so all
# blocks (including hits) are offloaded.
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
group_state.next_stored_block_idx = num_blocks
# Fence dst blocks against finished-request pending stores.
@@ -950,6 +956,12 @@ class OffloadingConnectorScheduler:
# TODO(orozery): possibly kickoff offload for last block
# which may have been deferred due to async scheduling
req_status = self._req_status.get(request.request_id)
req_context = (
req_status.req_context if req_status else _create_req_context(request)
)
self.manager.on_request_finished(req_context)
if req_status is None:
return False, None
if not req_status.transfer_jobs:
@@ -124,6 +124,10 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
return self.connector_worker.build_connector_worker_meta()
return None
def on_new_request(self, request: "Request") -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.on_new_request(request)
def get_num_new_matched_tokens(
self, request: "Request", num_computed_tokens: int
) -> tuple[int | None, bool]: