[KV Offload] Add per-request max_load_tokens control (#55885)

Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
alberto
2026-09-16 06:57:38 +03:00
committed by GitHub
parent 622934e61f
commit 75dc588269
3 changed files with 151 additions and 1 deletions
@@ -354,6 +354,8 @@ class RequestOffloadState:
req_context: ReqContext
offloading_context: RequestOffloadingContext
group_states: tuple[RequestGroupState, ...] = field(init=False)
# upper bound on tokens to load for this request; None means no cap
max_load_tokens: int | None = None
# upper bound on tokens to offload for this request; None means no cap
max_offload_tokens: int | None = None
# number of hits in the GPU cache
@@ -376,6 +378,20 @@ class RequestOffloadState:
)
params = self.req.kv_transfer_params
# NOTE: This field is experimental and subject to change in the future.
raw = params.get("max_load_tokens") if params else None
if type(raw) is int and raw >= 0:
self.max_load_tokens = raw
logger.debug(
"Request %s: max_load_tokens set to %d",
self.req.request_id,
raw,
)
elif raw is not None:
logger.warning(
"max_load_tokens must be a non-negative int, got %r; ignoring", raw
)
# NOTE: This field is experimental and subject to change in the future.
raw = params.get("max_offload_tokens") if params else None
if type(raw) is int and raw >= 0:
@@ -767,6 +783,11 @@ class OffloadingConnectorScheduler:
max_hit_size_tokens = min(
max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
)
if req_status.max_load_tokens is not None:
max_hit_size_tokens = min(
max_hit_size_tokens,
num_computed_tokens + req_status.max_load_tokens,
)
if self._sliding_window_groups:
# the last prompt token has to be recomputed to get the logprobs
# for sliding window attention, we must reduce by 1 to make sure
@@ -810,6 +831,10 @@ class OffloadingConnectorScheduler:
max_hit_size_tokens = min(
max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
)
if req_status.max_load_tokens is not None:
max_hit_size_tokens = round_down(
max_hit_size_tokens, tokens_per_chunk
)
if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
# We can only load less than a chunk, so skip.
return 0
@@ -965,6 +990,11 @@ class OffloadingConnectorScheduler:
max_boundary = min(req_status.req.num_prompt_tokens - 1, block_end - 1)
if max_num_new_tokens is not None:
max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
if req_status.max_load_tokens is not None:
max_boundary = min(
max_boundary,
local_tokens + req_status.max_load_tokens,
)
max_boundary = round_down(max_boundary, tokens_per_hash)
if max_boundary <= complete_boundary:
return complete_hit