[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Yifan Qiao
2026-10-03 06:26:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Codex
parent 1a001d5842
commit ca0df1f122
17 changed files with 162 additions and 72 deletions
@@ -111,6 +111,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
}
fail_scheduler.connector = Mock()
fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
)
@@ -327,6 +327,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
)
recompute_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
recompute_scheduler.connector.request_finished.return_value = (False, None)
recompute_scheduler.connector.take_events.return_value = ()
@@ -42,6 +42,7 @@ def test_failed_receive_completion_honors_failure_policy(policy):
request = create_request(num_tokens=3 * scheduler.block_size)
scheduler.add_request(request)
scheduler.connector = Mock()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = [
(2 * scheduler.block_size, True),
(0, False),
@@ -103,6 +104,7 @@ def test_async_load_failure(
}
scheduler.connector = Mock()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
)
@@ -332,6 +334,7 @@ def test_async_progressive_load_failure(
}
scheduler.connector = Mock()
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
scheduler.connector.get_num_new_matched_tokens.side_effect = (
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
)
@@ -202,6 +202,20 @@ def mc() -> MultiConnector:
return mc
@pytest.mark.parametrize("chosen", [0, 1])
def test_loaded_groups_follow_the_connector_serving_the_request(mc, chosen):
request = MagicMock(request_id="request")
for i, connector in enumerate(mc._connectors):
connector.get_num_new_matched_tokens.return_value = (
(16, True) if i == chosen else (0, False)
)
connector.get_loaded_kv_cache_group_ids.return_value = (i,)
assert mc.get_num_new_matched_tokens(request, 0) == (16, True)
assert mc.get_loaded_kv_cache_group_ids(request) == (chosen,)
mc._connectors[1 - chosen].get_loaded_kv_cache_group_ids.assert_not_called()
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
assert mc.get_mem_pool_context() is None
@@ -1522,34 +1522,6 @@ def _make_fake_kv_cache_manager():
return manager
@pytest.mark.cpu_test
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
"""Only zero-recorded (attention) groups contribute, sliced to the
externally-loaded token range; Mamba state blocks are never zeroed."""
manager = _make_fake_kv_cache_manager()
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
assert manager.get_zeroing_block_ids_in_range("req-1", 16, 56) == [11, 12, 13]
@pytest.mark.cpu_test
def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
"""Blocks that will be loaded by the connector must not be zeroed."""
from vllm.v1.core.sched.scheduler import Scheduler
class FakeKVCacheManager:
def take_new_block_ids(self):
return [9, 10, 11, 12]
scheduler = object.__new__(Scheduler)
scheduler.needs_kv_cache_zeroing = True
scheduler.kv_cache_manager = FakeKVCacheManager()
scheduler._skip_zero_block_ids = {10, 12}
assert scheduler._get_new_block_ids_to_zero() == [9, 11]
assert not scheduler._skip_zero_block_ids
@pytest.mark.cpu_test
def test_failed_load_rezeroes_unwritten_skipped_blocks():
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
@@ -1575,7 +1547,6 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
# Attention blocks covering tokens >= 48 are re-recorded for zeroing
# and flow into the next step's zero list; Mamba blocks are not.
scheduler._skip_zero_block_ids = set()
assert scheduler._get_new_block_ids_to_zero() == [13, 14, 15]