[Bugfix] Honor skip_reading_prefix_cache for KV connector hits (#57269)
Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com> Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
@@ -2302,6 +2302,25 @@ def test_has_sync_kv_loads(
|
||||
assert output.has_sync_kv_loads is expected_has_sync_loads
|
||||
|
||||
|
||||
def test_kv_connector_honors_skip_reading_prefix_cache():
|
||||
"""A request that must score every prompt row takes no external hit."""
|
||||
BLOCK_SIZE = 16
|
||||
scheduler = create_scheduler(
|
||||
enable_prefix_caching=True,
|
||||
use_kv_connector=mock_kv(matched_tokens=BLOCK_SIZE * 2, is_async=False),
|
||||
block_size=BLOCK_SIZE,
|
||||
)
|
||||
plain, scoring = create_requests(num_requests=2, num_tokens=BLOCK_SIZE * 4)
|
||||
scoring.sampling_params.prompt_logprobs = 1
|
||||
scoring.skip_reading_prefix_cache = True
|
||||
scheduler.add_request(plain)
|
||||
scheduler.add_request(scoring)
|
||||
|
||||
output = scheduler.schedule()
|
||||
assert output.num_scheduled_tokens[plain.request_id] == BLOCK_SIZE * 2
|
||||
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [False, True])
|
||||
def test_kv_connector_basic(is_async: bool):
|
||||
"""Test whether Scheduler with KVConnector schedules tokens, allocates
|
||||
|
||||
@@ -935,6 +935,8 @@ class Scheduler(SchedulerInterface):
|
||||
request, block_aligned_local
|
||||
)
|
||||
)
|
||||
if request.skip_reading_prefix_cache:
|
||||
ext_tokens, load_kv_async = 0, False
|
||||
|
||||
if ext_tokens is None:
|
||||
# The request cannot be scheduled because
|
||||
|
||||
Reference in New Issue
Block a user