[Bugfix] Honor skip_reading_prefix_cache for KV connector hits (#57269)

Signed-off-by: aoshen02 <aoshen02@users.noreply.github.com>
Co-authored-by: aoshen02 <aoshen02@users.noreply.github.com>
This commit is contained in:
aoshen02
2026-09-17 03:46:28 -07:00
committed by GitHub
co-authored by aoshen02
parent 0d72627565
commit 9ff08f9493
2 changed files with 21 additions and 0 deletions
+19
View File
@@ -2302,6 +2302,25 @@ def test_has_sync_kv_loads(
assert output.has_sync_kv_loads is expected_has_sync_loads
def test_kv_connector_honors_skip_reading_prefix_cache():
"""A request that must score every prompt row takes no external hit."""
BLOCK_SIZE = 16
scheduler = create_scheduler(
enable_prefix_caching=True,
use_kv_connector=mock_kv(matched_tokens=BLOCK_SIZE * 2, is_async=False),
block_size=BLOCK_SIZE,
)
plain, scoring = create_requests(num_requests=2, num_tokens=BLOCK_SIZE * 4)
scoring.sampling_params.prompt_logprobs = 1
scoring.skip_reading_prefix_cache = True
scheduler.add_request(plain)
scheduler.add_request(scoring)
output = scheduler.schedule()
assert output.num_scheduled_tokens[plain.request_id] == BLOCK_SIZE * 2
assert output.num_scheduled_tokens[scoring.request_id] == BLOCK_SIZE * 4
@pytest.mark.parametrize("is_async", [False, True])
def test_kv_connector_basic(is_async: bool):
"""Test whether Scheduler with KVConnector schedules tokens, allocates
+2
View File
@@ -935,6 +935,8 @@ class Scheduler(SchedulerInterface):
request, block_aligned_local
)
)
if request.skip_reading_prefix_cache:
ext_tokens, load_kv_async = 0, False
if ext_tokens is None:
# The request cannot be scheduled because