[KV Offload] Add per-request max_load_tokens control (#55885)

Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
alberto
2026-09-16 06:57:38 +03:00
committed by GitHub
parent 622934e61f
commit 75dc588269
3 changed files with 151 additions and 1 deletions
+29
View File
@@ -22,6 +22,35 @@ flowchart LR
CPU <--> SN["..."] CPU <--> SN["..."]
``` ```
## Per-request load control
Individual requests can cap how many tokens are loaded from offloaded storage
by setting `max_load_tokens` in `kv_transfer_params`. The cap applies to tokens
beyond those already available in the GPU prefix cache. Set it to `0` to
disable external loading for the request:
```json
{
"kv_transfer_params": {
"max_load_tokens": 0
}
}
```
GPU prefix-cache reuse remains enabled, and tokens beyond the aligned load cap
are recomputed. Omitting the field leaves loading uncapped. The value must be a
non-negative integer; invalid values are ignored. Positive caps are rounded
down to a boundary supported by the configured KV cache groups. Offloading
remains enabled unless it is controlled separately with `max_offload_tokens`.
!!! warning
`max_load_tokens` is experimental and subject to change.
`kv_load_tiers` continues to select secondary tiers. CPU is always included:
it can satisfy a resident hit directly and is the required staging tier for
secondary-to-GPU loads. Consequently, an empty tier list allows CPU hits but
does not query any secondary tier.
## Terminology: Chunks ## Terminology: Chunks
The unit of operation is a **chunk** — a fixed-size piece of KV data covering a group of tokens. By default, a chunk maps to a single accelerator block. A configurable `blocks_per_chunk` parameter allows larger chunks, yielding larger I/Os to the host and secondary tiers. The unit of operation is a **chunk** — a fixed-size piece of KV data covering a group of tokens. By default, a chunk maps to a single accelerator block. A configurable `blocks_per_chunk` parameter allows larger chunks, yielding larger I/Os to the host and secondary tiers.
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock, call from unittest.mock import MagicMock, call
import pytest import pytest
@@ -195,15 +196,17 @@ def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
def _make_partial_tail_request( def _make_partial_tail_request(
scheduler: OffloadingConnectorScheduler, scheduler: OffloadingConnectorScheduler,
kv_transfer_params: dict[str, Any] | None = None,
) -> MagicMock: ) -> MagicMock:
request = MagicMock() request = MagicMock()
request.request_id = "req" request.request_id = "req"
request.kv_transfer_params = None request.kv_transfer_params = kv_transfer_params
request.num_prompt_tokens = 30 request.num_prompt_tokens = 30
request.num_tokens = 30 request.num_tokens = 30
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)] request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
request.all_token_ids = list(range(30)) request.all_token_ids = list(range(30))
request.lora_request = None request.lora_request = None
request.skip_reading_prefix_cache = False
request.is_finished.return_value = False request.is_finished.return_value = False
scheduler.on_new_request(request) scheduler.on_new_request(request)
return request return request
@@ -427,6 +430,53 @@ def test_lookup_cap_stops_at_authoritative_prefix_boundary():
assert scheduler._req_status["req"].partial_tail_boundary == 20 assert scheduler._req_status["req"].partial_tail_boundary == 20
def test_max_load_tokens_zero_skips_partial_tail_lookup():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 0}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 16) == (0, False)
scheduler.manager.lookup.assert_not_called()
assert all(
state.num_hit_chunks == 1 for state in scheduler._req_status["req"].group_states
)
def test_max_load_tokens_caps_tokens_beyond_gpu_prefix():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 8}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 16) == (8, True)
assert scheduler._req_status["req"].partial_tail_boundary == 24
def test_max_load_tokens_rounds_down_without_partial_tail():
scheduler = _make_partial_tail_scheduler()
scheduler.config = scheduler.config._replace(supports_partial_tail=False)
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 20}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 0) == (16, True)
@pytest.mark.parametrize("value", ["8", 8.5, -1, True])
def test_invalid_max_load_tokens_is_ignored(value):
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": value}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 0) == (28, True)
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary(): def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
"""A recurrent group may hold unhashed blocks inside the computed prefix. """A recurrent group may hold unhashed blocks inside the computed prefix.
@@ -2964,6 +3014,47 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
runner.manager.lookup.assert_not_called() runner.manager.lookup.assert_not_called()
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_max_load_tokens_limits_external_load(request_runner, async_scheduling: bool):
"""The load cap limits external reads without changing the store path."""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
blocks_per_chunk=blocks_per_chunk,
)
runner.new_request(token_ids=[0] * tokens_per_chunk)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=(0, 1, 2),
)
runner.scheduler.reset_prefix_cache()
runner.manager.lookup.reset_mock()
runner.new_request(
token_ids=[0] * tokens_per_chunk,
kv_transfer_params={"max_load_tokens": 0},
)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_loaded=(),
expected_stored=(0, 1, 2),
)
runner.manager.lookup.assert_not_called()
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
# Eagle/MTP test class # Eagle/MTP test class
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@@ -354,6 +354,8 @@ class RequestOffloadState:
req_context: ReqContext req_context: ReqContext
offloading_context: RequestOffloadingContext offloading_context: RequestOffloadingContext
group_states: tuple[RequestGroupState, ...] = field(init=False) group_states: tuple[RequestGroupState, ...] = field(init=False)
# upper bound on tokens to load for this request; None means no cap
max_load_tokens: int | None = None
# upper bound on tokens to offload for this request; None means no cap # upper bound on tokens to offload for this request; None means no cap
max_offload_tokens: int | None = None max_offload_tokens: int | None = None
# number of hits in the GPU cache # number of hits in the GPU cache
@@ -376,6 +378,20 @@ class RequestOffloadState:
) )
params = self.req.kv_transfer_params params = self.req.kv_transfer_params
# NOTE: This field is experimental and subject to change in the future.
raw = params.get("max_load_tokens") if params else None
if type(raw) is int and raw >= 0:
self.max_load_tokens = raw
logger.debug(
"Request %s: max_load_tokens set to %d",
self.req.request_id,
raw,
)
elif raw is not None:
logger.warning(
"max_load_tokens must be a non-negative int, got %r; ignoring", raw
)
# NOTE: This field is experimental and subject to change in the future. # NOTE: This field is experimental and subject to change in the future.
raw = params.get("max_offload_tokens") if params else None raw = params.get("max_offload_tokens") if params else None
if type(raw) is int and raw >= 0: if type(raw) is int and raw >= 0:
@@ -767,6 +783,11 @@ class OffloadingConnectorScheduler:
max_hit_size_tokens = min( max_hit_size_tokens = min(
max_hit_size_tokens, num_computed_tokens + max_num_new_tokens max_hit_size_tokens, num_computed_tokens + max_num_new_tokens
) )
if req_status.max_load_tokens is not None:
max_hit_size_tokens = min(
max_hit_size_tokens,
num_computed_tokens + req_status.max_load_tokens,
)
if self._sliding_window_groups: if self._sliding_window_groups:
# the last prompt token has to be recomputed to get the logprobs # the last prompt token has to be recomputed to get the logprobs
# for sliding window attention, we must reduce by 1 to make sure # for sliding window attention, we must reduce by 1 to make sure
@@ -810,6 +831,10 @@ class OffloadingConnectorScheduler:
max_hit_size_tokens = min( max_hit_size_tokens = min(
max_hit_size_tokens, len(offload_keys) * tokens_per_chunk max_hit_size_tokens, len(offload_keys) * tokens_per_chunk
) )
if req_status.max_load_tokens is not None:
max_hit_size_tokens = round_down(
max_hit_size_tokens, tokens_per_chunk
)
if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk: if max_hit_size_tokens - num_computed_tokens < tokens_per_chunk:
# We can only load less than a chunk, so skip. # We can only load less than a chunk, so skip.
return 0 return 0
@@ -965,6 +990,11 @@ class OffloadingConnectorScheduler:
max_boundary = min(req_status.req.num_prompt_tokens - 1, block_end - 1) max_boundary = min(req_status.req.num_prompt_tokens - 1, block_end - 1)
if max_num_new_tokens is not None: if max_num_new_tokens is not None:
max_boundary = min(max_boundary, local_tokens + max_num_new_tokens) max_boundary = min(max_boundary, local_tokens + max_num_new_tokens)
if req_status.max_load_tokens is not None:
max_boundary = min(
max_boundary,
local_tokens + req_status.max_load_tokens,
)
max_boundary = round_down(max_boundary, tokens_per_hash) max_boundary = round_down(max_boundary, tokens_per_hash)
if max_boundary <= complete_boundary: if max_boundary <= complete_boundary:
return complete_hit return complete_hit