[KV Offload] Add per-request max_load_tokens control (#55885)
Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
@@ -354,6 +354,8 @@ class RequestOffloadState:
|
||||
req_context: ReqContext
|
||||
offloading_context: RequestOffloadingContext
|
||||
group_states: tuple[RequestGroupState, ...] = field(init=False)
|
||||
# upper bound on tokens to load for this request; None means no cap
|
||||
max_load_tokens: int | None = None
|
||||
# upper bound on tokens to offload for this request; None means no cap
|
||||
max_offload_tokens: int | None = None
|
||||
# number of hits in the GPU cache
|
||||
@@ -376,6 +378,20 @@ class RequestOffloadState:
|
||||
)
|
||||
params = self.req.kv_transfer_params
|
||||
|
||||
# NOTE: This field is experimental and subject to change in the future.
|
||||
raw = params.get("max_load_tokens") if params else None
|
||||
if type(raw) is int and raw >= 0:
|
||||
self.max_load_tokens = raw
|
||||
logger.debug(
|
||||
"Request %s: max_load_tokens set to %d",
|
||||
self.req.request_id,
|
||||
raw,
|
||||
)
|
||||
elif raw is not None:
|
||||
logger.warning(
|
||||
"max_load_tokens must be a non-negative int, got %r; ignoring", raw
|
||||
)
|
||||
|
||||
# NOTE: This field is experimental and subject to change in the future.
|
||||
raw = params.get("max_offload_tokens") if params else None
|
||||
if type(raw) is int and raw >= 0:
|
||||
@@ -767,6 +783,11 @@ class OffloadingConnectorScheduler:
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
|
||||
)
|
||||
if req_status.max_load_tokens is not None:
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens,
|
||||
num_computed_tokens + req_status.max_load_tokens,
|
||||
)
|
||||
if self._sliding_window_groups:
|
||||
# the last prompt token has to be recomputed to get the logprobs
|
||||
# for sliding window attention, we must reduce by 1 to make sure
|
||||
@@ -810,6 +831,10 @@ class OffloadingConnectorScheduler:
|
||||
max_hit_size_tokens = min(
|
||||
max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
|
||||
)
|
||||
if req_status.max_load_tokens is not None:
|
||||
max_hit_size_tokens = round_down(
|
||||
max_hit_size_tokens, tokens_per_chunk
|
||||
)
|
||||
if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
|
||||
# We can only load less than a chunk, so skip.
|
||||
return 0
|
||||
@@ -965,6 +990,11 @@ class OffloadingConnectorScheduler:
|
||||
max_boundary = min(req_status.req.num_prompt_tokens - 1, block_end - 1)
|
||||
if max_num_new_tokens is not None:
|
||||
max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
|
||||
if req_status.max_load_tokens is not None:
|
||||
max_boundary = min(
|
||||
max_boundary,
|
||||
local_tokens + req_status.max_load_tokens,
|
||||
)
|
||||
max_boundary = round_down(max_boundary, tokens_per_hash)
|
||||
if max_boundary <= complete_boundary:
|
||||
return complete_hit
|
||||
|
||||
Reference in New Issue
Block a user