[KV Offload] Add per-request offloading policy via on_new_request lifecycle hook (#43205)
Signed-off-by: Ronen Schaffer <ronen.schaffer@ibm.com> Co-authored-by: Or Ozeri <or@ozery.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
Or Ozeri
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
7e53283b1c
commit
a3ed5ab10c
@@ -29,7 +29,9 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingManager,
|
||||
OffloadingSpec,
|
||||
OffloadKey,
|
||||
OffloadPolicy,
|
||||
ReqContext,
|
||||
RequestOffloadingContext,
|
||||
get_offload_block_hash,
|
||||
make_offload_key,
|
||||
)
|
||||
@@ -172,8 +174,9 @@ class RequestGroupState:
|
||||
class RequestOffloadState:
|
||||
config: SchedulerOffloadConfig
|
||||
req: Request
|
||||
req_context: ReqContext
|
||||
offloading_context: RequestOffloadingContext
|
||||
group_states: tuple[RequestGroupState, ...] = field(init=False)
|
||||
req_context: ReqContext = field(init=False)
|
||||
# upper bound on tokens to offload for this request; None means no cap
|
||||
max_offload_tokens: int | None = None
|
||||
# number of hits in the GPU cache
|
||||
@@ -186,10 +189,6 @@ class RequestOffloadState:
|
||||
self.group_states = tuple(
|
||||
RequestGroupState() for _ in self.config.kv_group_configs
|
||||
)
|
||||
self.req_context = ReqContext(
|
||||
req_id=self.req.request_id,
|
||||
kv_transfer_params=self.req.kv_transfer_params,
|
||||
)
|
||||
params = self.req.kv_transfer_params
|
||||
|
||||
# NOTE: This field is experimental and subject to change in the future.
|
||||
@@ -248,6 +247,13 @@ class RequestOffloadState:
|
||||
)
|
||||
|
||||
|
||||
def _create_req_context(req: Request) -> ReqContext:
|
||||
return ReqContext(
|
||||
req_id=req.request_id,
|
||||
kv_transfer_params=req.kv_transfer_params,
|
||||
)
|
||||
|
||||
|
||||
class OffloadingConnectorScheduler:
|
||||
"""Implementation of Scheduler side methods"""
|
||||
|
||||
@@ -512,6 +518,18 @@ class OffloadingConnectorScheduler:
|
||||
|
||||
return num_hit_tokens
|
||||
|
||||
def on_new_request(self, request: Request) -> None:
|
||||
"""Called when a new request is added to the scheduler."""
|
||||
req_context = _create_req_context(request)
|
||||
offloading_context = self.manager.on_new_request(req_context)
|
||||
req_status = RequestOffloadState(
|
||||
config=self.config,
|
||||
req=request,
|
||||
req_context=req_context,
|
||||
offloading_context=offloading_context,
|
||||
)
|
||||
self._req_status[request.request_id] = req_status
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: Request, num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
@@ -534,24 +552,15 @@ class OffloadingConnectorScheduler:
|
||||
- `True` if tokens will be loaded asynchronously
|
||||
(between scheduler steps).
|
||||
"""
|
||||
is_new_request = False
|
||||
if req_status := self._req_status.get(request.request_id):
|
||||
# make sure block IDs are cleared
|
||||
for group_state in req_status.group_states:
|
||||
group_state.block_ids.clear()
|
||||
else:
|
||||
is_new_request = True
|
||||
req_status = RequestOffloadState(config=self.config, req=request)
|
||||
self._req_status[request.request_id] = req_status
|
||||
req_status = self._req_status[request.request_id]
|
||||
for group_state in req_status.group_states:
|
||||
group_state.block_ids.clear()
|
||||
|
||||
req_status.update_offload_keys()
|
||||
req_status.num_locally_computed_tokens = num_computed_tokens
|
||||
|
||||
num_hit_tokens = self._lookup(req_status)
|
||||
if is_new_request:
|
||||
req_status.update_num_hit_blocks(
|
||||
num_computed_tokens + (num_hit_tokens or 0)
|
||||
)
|
||||
req_status.update_num_hit_blocks(num_computed_tokens + (num_hit_tokens or 0))
|
||||
|
||||
self._touch(req_status)
|
||||
|
||||
@@ -568,9 +577,6 @@ class OffloadingConnectorScheduler:
|
||||
num_locally_computed_tokens = req_status.num_locally_computed_tokens
|
||||
num_cached_tokens = num_locally_computed_tokens + num_external_tokens
|
||||
|
||||
params = req_status.req_context.kv_transfer_params
|
||||
do_remote_decode = params is not None and params.get("do_remote_decode")
|
||||
|
||||
keys_to_load: list[OffloadKey] = []
|
||||
dst_block_ids: list[int] = []
|
||||
# per group
|
||||
@@ -624,10 +630,10 @@ class OffloadingConnectorScheduler:
|
||||
group_sizes.append(num_pending_gpu_blocks)
|
||||
block_indices.append(num_locally_computed_gpu_blocks)
|
||||
|
||||
if not do_remote_decode:
|
||||
# For P/D prefill requests (do_remote_decode=True), we do
|
||||
# NOT skip saving the hit prefix, as we need to stream the
|
||||
# entire KV cache so a remote decode node can consume it.
|
||||
# Skip prefix-hit blocks for block-level policy; for
|
||||
# request-level, next_stored_block_idx stays at 0 so all
|
||||
# blocks (including hits) are offloaded.
|
||||
if req_status.offloading_context.policy == OffloadPolicy.BLOCK_LEVEL:
|
||||
group_state.next_stored_block_idx = num_blocks
|
||||
|
||||
# Fence dst blocks against finished-request pending stores.
|
||||
@@ -950,6 +956,12 @@ class OffloadingConnectorScheduler:
|
||||
# TODO(orozery): possibly kickoff offload for last block
|
||||
# which may have been deferred due to async scheduling
|
||||
req_status = self._req_status.get(request.request_id)
|
||||
|
||||
req_context = (
|
||||
req_status.req_context if req_status else _create_req_context(request)
|
||||
)
|
||||
self.manager.on_request_finished(req_context)
|
||||
|
||||
if req_status is None:
|
||||
return False, None
|
||||
if not req_status.transfer_jobs:
|
||||
|
||||
@@ -124,6 +124,10 @@ class OffloadingConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
return None
|
||||
|
||||
def on_new_request(self, request: "Request") -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.on_new_request(request)
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self, request: "Request", num_computed_tokens: int
|
||||
) -> tuple[int | None, bool]:
|
||||
|
||||
Reference in New Issue
Block a user