[KV Offload] Add per-request max_load_tokens control (#55885)
Signed-off-by: Alberto Perdomo <aperdomo@redhat.com>
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
@@ -195,15 +196,17 @@ def _make_partial_tail_scheduler() -> OffloadingConnectorScheduler:
|
||||
|
||||
def _make_partial_tail_request(
|
||||
scheduler: OffloadingConnectorScheduler,
|
||||
kv_transfer_params: dict[str, Any] | None = None,
|
||||
) -> MagicMock:
|
||||
request = MagicMock()
|
||||
request.request_id = "req"
|
||||
request.kv_transfer_params = None
|
||||
request.kv_transfer_params = kv_transfer_params
|
||||
request.num_prompt_tokens = 30
|
||||
request.num_tokens = 30
|
||||
request.block_hashes = [BlockHash(f"h{i}".encode()) for i in range(7)]
|
||||
request.all_token_ids = list(range(30))
|
||||
request.lora_request = None
|
||||
request.skip_reading_prefix_cache = False
|
||||
request.is_finished.return_value = False
|
||||
scheduler.on_new_request(request)
|
||||
return request
|
||||
@@ -427,6 +430,53 @@ def test_lookup_cap_stops_at_authoritative_prefix_boundary():
|
||||
assert scheduler._req_status["req"].partial_tail_boundary == 20
|
||||
|
||||
|
||||
def test_max_load_tokens_zero_skips_partial_tail_lookup():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": 0}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 16) == (0, False)
|
||||
scheduler.manager.lookup.assert_not_called()
|
||||
assert all(
|
||||
state.num_hit_chunks == 1 for state in scheduler._req_status["req"].group_states
|
||||
)
|
||||
|
||||
|
||||
def test_max_load_tokens_caps_tokens_beyond_gpu_prefix():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": 8}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 16) == (8, True)
|
||||
assert scheduler._req_status["req"].partial_tail_boundary == 24
|
||||
|
||||
|
||||
def test_max_load_tokens_rounds_down_without_partial_tail():
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
scheduler.config = scheduler.config._replace(supports_partial_tail=False)
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": 20}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == (16, True)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("value", ["8", 8.5, -1, True])
|
||||
def test_invalid_max_load_tokens_is_ignored(value):
|
||||
scheduler = _make_partial_tail_scheduler()
|
||||
request = _make_partial_tail_request(
|
||||
scheduler, kv_transfer_params={"max_load_tokens": value}
|
||||
)
|
||||
scheduler.manager.lookup.return_value = LookupResult.HIT
|
||||
|
||||
assert scheduler.get_num_new_matched_tokens(request, 0) == (28, True)
|
||||
|
||||
|
||||
def test_recurrent_group_unhashed_block_does_not_truncate_load_boundary():
|
||||
"""A recurrent group may hold unhashed blocks inside the computed prefix.
|
||||
|
||||
@@ -2964,6 +3014,47 @@ def test_skip_reading_prefix_cache(request_runner, async_scheduling: bool):
|
||||
runner.manager.lookup.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("async_scheduling", [True, False])
|
||||
def test_max_load_tokens_limits_external_load(request_runner, async_scheduling: bool):
|
||||
"""The load cap limits external reads without changing the store path."""
|
||||
block_size = 4
|
||||
blocks_per_chunk = 3
|
||||
tokens_per_chunk = block_size * blocks_per_chunk
|
||||
|
||||
runner = request_runner(
|
||||
block_size=block_size,
|
||||
num_gpu_blocks=100,
|
||||
async_scheduling=async_scheduling,
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
)
|
||||
|
||||
runner.new_request(token_ids=[0] * tokens_per_chunk)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_stored=(0, 1, 2),
|
||||
)
|
||||
|
||||
runner.scheduler.reset_prefix_cache()
|
||||
runner.manager.lookup.reset_mock()
|
||||
runner.new_request(
|
||||
token_ids=[0] * tokens_per_chunk,
|
||||
kv_transfer_params={"max_load_tokens": 0},
|
||||
)
|
||||
runner.manager.prepare_store.side_effect = lambda keys, req_context: (
|
||||
generate_store_output(keys)
|
||||
)
|
||||
runner.run(
|
||||
decoded_tokens=[EOS_TOKEN_ID],
|
||||
expected_loaded=(),
|
||||
expected_stored=(0, 1, 2),
|
||||
)
|
||||
|
||||
runner.manager.lookup.assert_not_called()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Eagle/MTP test class
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
Reference in New Issue
Block a user