[KV Offload] Add per-request max_load_tokens control (#55885)

Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
alberto
2026-09-16 06:57:38 +03:00
committed by GitHub
parent 622934e61f
commit 75dc588269
3 changed files with 151 additions and 1 deletions
@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock, call
import pytest
@@ -195,15 +196,17 @@ def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
def _make_partial_tail_request(
scheduler: OffloadingConnectorScheduler,
kv_transfer_params: dict[str, Any] | None = None,
) -> MagicMock:
request = MagicMock()
request.request_id = "req"
request.kv_transfer_params = None
request.kv_transfer_params = kv_transfer_params
request.num_prompt_tokens = 30
request.num_tokens = 30
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
request.all_token_ids = list(range(30))
request.lora_request = None
request.skip_reading_prefix_cache = False
request.is_finished.return_value = False
scheduler.on_new_request(request)
return request
@@ -427,6 +430,53 @@ def test_lookup_cap_stops_at_authoritative_prefix_boundary():
assert scheduler._req_status["req"].partial_tail_boundary == 20
def test_max_load_tokens_zero_skips_partial_tail_lookup():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 0}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 16) == (0, False)
scheduler.manager.lookup.assert_not_called()
assert all(
state.num_hit_chunks == 1 for state in scheduler._req_status["req"].group_states
)
def test_max_load_tokens_caps_tokens_beyond_gpu_prefix():
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 8}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 16) == (8, True)
assert scheduler._req_status["req"].partial_tail_boundary == 24
def test_max_load_tokens_rounds_down_without_partial_tail():
scheduler = _make_partial_tail_scheduler()
scheduler.config = scheduler.config._replace(supports_partial_tail=False)
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": 20}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 0) == (16, True)
@pytest.mark.parametrize("value", ["8", 8.5, -1, True])
def test_invalid_max_load_tokens_is_ignored(value):
scheduler = _make_partial_tail_scheduler()
request = _make_partial_tail_request(
scheduler, kv_transfer_params={"max_load_tokens": value}
)
scheduler.manager.lookup.return_value = LookupResult.HIT
assert scheduler.get_num_new_matched_tokens(request, 0) == (28, True)
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
"""A recurrent group may hold unhashed blocks inside the computed prefix.
@@ -2964,6 +3014,47 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
runner.manager.lookup.assert_not_called()
@pytest.mark.parametrize("async_scheduling", [True, False])
def test_max_load_tokens_limits_external_load(request_runner, async_scheduling: bool):
"""The load cap limits external reads without changing the store path."""
block_size = 4
blocks_per_chunk = 3
tokens_per_chunk = block_size * blocks_per_chunk
runner = request_runner(
block_size=block_size,
num_gpu_blocks=100,
async_scheduling=async_scheduling,
blocks_per_chunk=blocks_per_chunk,
)
runner.new_request(token_ids=[0] * tokens_per_chunk)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_stored=(0, 1, 2),
)
runner.scheduler.reset_prefix_cache()
runner.manager.lookup.reset_mock()
runner.new_request(
token_ids=[0] * tokens_per_chunk,
kv_transfer_params={"max_load_tokens": 0},
)
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
generate_store_output(keys)
)
runner.run(
decoded_tokens=[EOS_TOKEN_ID],
expected_loaded=(),
expected_stored=(0, 1, 2),
)
runner.manager.lookup.assert_not_called()
# ---------------------------------------------------------------------------
# Eagle/MTP test class
# ---------------------------------------------------------------------------