[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Codex
parent
1a001d5842
commit
ca0df1f122
@@ -4,6 +4,8 @@
|
||||
the hit to rebuild the non-cacheable sliding-window group, keeps the hit's
|
||||
blocks, and hands the worker the replay start."""
|
||||
|
||||
import dataclasses
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -277,3 +279,53 @@ def test_hit_no_longer_than_window_is_ignored(via_connector):
|
||||
assert new_req.num_computed_tokens == 0
|
||||
assert new_req.replay_start == 0
|
||||
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS
|
||||
|
||||
|
||||
@pytest.mark.parametrize("is_async", [True, False])
|
||||
@pytest.mark.parametrize("loads_window", [True, False])
|
||||
def test_kv_load_zeroing_respects_loaded_groups(is_async, loads_window):
|
||||
"""Only groups restored by an async load skip zeroing. P/D loads include
|
||||
the non-cacheable window; stores leave it to be initialized locally."""
|
||||
local = 5 * BLOCK_SIZE
|
||||
scheduler = _replay_scheduler(use_kv_connector=MockKVConfig())
|
||||
if loads_window:
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
|
||||
assert scheduler.needs_kv_cache_zeroing
|
||||
first = create_requests(
|
||||
num_requests=1, num_tokens=local, same_prompt=True, block_size=BLOCK_SIZE
|
||||
)[0]
|
||||
_prefill(scheduler, first)
|
||||
scheduler.connector.config = dataclasses.replace(
|
||||
scheduler.connector.config,
|
||||
matched_tokens=NUM_PROMPT_TOKENS - 1 - local,
|
||||
is_async=is_async,
|
||||
)
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=NUM_PROMPT_TOKENS,
|
||||
same_prompt=True,
|
||||
block_size=BLOCK_SIZE,
|
||||
req_ids=["load"],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
out = scheduler.schedule()
|
||||
|
||||
manager = scheduler.kv_cache_manager
|
||||
hit_ids = {
|
||||
b.block_id for g in manager.get_blocks(first.request_id).blocks for b in g
|
||||
}
|
||||
swa_blocks = manager.get_blocks(request.request_id).blocks[SWA]
|
||||
assert any(
|
||||
not b.is_null and b.block_id not in hit_ids
|
||||
for b in swa_blocks[: local // BLOCK_SIZE]
|
||||
)
|
||||
zeroed = set(out.new_block_ids_to_zero or ())
|
||||
for group_id, blocks in enumerate(manager.get_blocks(request.request_id).blocks):
|
||||
new_ids = {
|
||||
b.block_id for b in blocks if not b.is_null and b.block_id not in hit_ids
|
||||
}
|
||||
assert new_ids
|
||||
if is_async and (group_id == FULL or loads_window):
|
||||
assert not new_ids & zeroed
|
||||
else:
|
||||
assert new_ids <= zeroed
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
|
||||
ChunkedLocalAttentionManager,
|
||||
CircularBufferManager,
|
||||
FullAttentionManager,
|
||||
KpoolTailManager,
|
||||
MambaManager,
|
||||
RSWAManager,
|
||||
SlidingWindowManager,
|
||||
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_interface import (
|
||||
ChunkedLocalAttentionSpec,
|
||||
CircularBufferSpec,
|
||||
FullAttentionSpec,
|
||||
KpoolTailSpec,
|
||||
MambaSpec,
|
||||
RSWASpec,
|
||||
SlidingWindowSpec,
|
||||
@@ -332,6 +334,37 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
|
||||
assert manager.req_to_blocks[request_id] == blocks
|
||||
|
||||
|
||||
@pytest.mark.parametrize("record_for_zeroing", [True, False])
|
||||
def test_external_kpool_tail_zeroing(record_for_zeroing):
|
||||
"""A transferred ring at index zero skips zeroing even with a local hit."""
|
||||
spec = KpoolTailSpec(
|
||||
block_size=4,
|
||||
num_kv_heads=2,
|
||||
head_size=8,
|
||||
dtype=torch.bfloat16,
|
||||
sliding_window=4,
|
||||
)
|
||||
manager = KpoolTailManager(
|
||||
spec,
|
||||
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
|
||||
enable_caching=True,
|
||||
kv_cache_group_id=0,
|
||||
scheduler_block_size=4,
|
||||
needs_kv_cache_zeroing=True,
|
||||
max_admission_blocks_per_request=1,
|
||||
)
|
||||
manager.allocate_external_computed_blocks(
|
||||
"request", 16, 4, record_for_zeroing=record_for_zeroing
|
||||
)
|
||||
blocks = manager.req_to_blocks["request"]
|
||||
assert len(blocks) == 1
|
||||
assert manager.take_new_block_ids() == (
|
||||
[blocks[0].block_id] if record_for_zeroing else []
|
||||
)
|
||||
assert manager.allocate_new_blocks("request", 20, 20) == []
|
||||
assert manager.take_new_block_ids() == []
|
||||
|
||||
|
||||
def test_sliding_window_records_new_blocks_for_zeroing():
|
||||
block_size = 2
|
||||
spec = SlidingWindowSpec(
|
||||
|
||||
@@ -192,6 +192,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
|
||||
full_sequence_must_fit: bool = False,
|
||||
reserved_blocks: int = 0,
|
||||
has_scheduled_reqs: bool = True,
|
||||
skip_zeroing_group_ids: tuple[int, ...] = (),
|
||||
):
|
||||
ret = original_allocate_slots_fn(
|
||||
self,
|
||||
@@ -206,6 +207,7 @@ def get_fake_allocate_slots_fn(original_allocate_slots_fn: Callable):
|
||||
full_sequence_must_fit,
|
||||
reserved_blocks,
|
||||
has_scheduled_reqs,
|
||||
skip_zeroing_group_ids,
|
||||
)
|
||||
if cur_step_action is not None:
|
||||
cur_block_ids = self.coordinator.single_type_managers[0].req_to_blocks[
|
||||
|
||||
@@ -111,6 +111,7 @@ def test_error_propagation_async_load(fail_scheduler: Scheduler):
|
||||
}
|
||||
|
||||
fail_scheduler.connector = Mock()
|
||||
fail_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
fail_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
|
||||
)
|
||||
|
||||
@@ -327,6 +327,7 @@ def test_async_recompute_blocks_not_cached_when_invalid(
|
||||
recompute_scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, True)
|
||||
)
|
||||
recompute_scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
recompute_scheduler.connector.request_finished.return_value = (False, None)
|
||||
recompute_scheduler.connector.take_events.return_value = ()
|
||||
|
||||
|
||||
@@ -42,6 +42,7 @@ def test_failed_receive_completion_honors_failure_policy(policy):
|
||||
request = create_request(num_tokens=3 * scheduler.block_size)
|
||||
scheduler.add_request(request)
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = [
|
||||
(2 * scheduler.block_size, True),
|
||||
(0, False),
|
||||
@@ -103,6 +104,7 @@ def test_async_load_failure(
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
|
||||
)
|
||||
@@ -332,6 +334,7 @@ def test_async_progressive_load_failure(
|
||||
}
|
||||
|
||||
scheduler.connector = Mock()
|
||||
scheduler.connector.get_loaded_kv_cache_group_ids.return_value = (0,)
|
||||
scheduler.connector.get_num_new_matched_tokens.side_effect = (
|
||||
_make_get_num_new_matched_tokens(req_num_new_matched_tokens, async_load=True)
|
||||
)
|
||||
|
||||
@@ -202,6 +202,20 @@ def mc() -> MultiConnector:
|
||||
return mc
|
||||
|
||||
|
||||
@pytest.mark.parametrize("chosen", [0, 1])
|
||||
def test_loaded_groups_follow_the_connector_serving_the_request(mc, chosen):
|
||||
request = MagicMock(request_id="request")
|
||||
for i, connector in enumerate(mc._connectors):
|
||||
connector.get_num_new_matched_tokens.return_value = (
|
||||
(16, True) if i == chosen else (0, False)
|
||||
)
|
||||
connector.get_loaded_kv_cache_group_ids.return_value = (i,)
|
||||
|
||||
assert mc.get_num_new_matched_tokens(request, 0) == (16, True)
|
||||
assert mc.get_loaded_kv_cache_group_ids(request) == (chosen,)
|
||||
mc._connectors[1 - chosen].get_loaded_kv_cache_group_ids.assert_not_called()
|
||||
|
||||
|
||||
def test_multi_connector_mem_pool_context_none(mc: MultiConnector):
|
||||
assert mc.get_mem_pool_context() is None
|
||||
|
||||
|
||||
@@ -1522,34 +1522,6 @@ def _make_fake_kv_cache_manager():
|
||||
return manager
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_zeroing_block_ids_cover_only_loaded_attention_blocks():
|
||||
"""Only zero-recorded (attention) groups contribute, sliced to the
|
||||
externally-loaded token range; Mamba state blocks are never zeroed."""
|
||||
manager = _make_fake_kv_cache_manager()
|
||||
|
||||
# Tokens [0, 16) are locally cached; the load covers tokens [16, 56).
|
||||
assert manager.get_zeroing_block_ids_in_range("req-1", 16, 56) == [11, 12, 13]
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_scheduler_filters_connector_loaded_blocks_from_zeroing():
|
||||
"""Blocks that will be loaded by the connector must not be zeroed."""
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
|
||||
class FakeKVCacheManager:
|
||||
def take_new_block_ids(self):
|
||||
return [9, 10, 11, 12]
|
||||
|
||||
scheduler = object.__new__(Scheduler)
|
||||
scheduler.needs_kv_cache_zeroing = True
|
||||
scheduler.kv_cache_manager = FakeKVCacheManager()
|
||||
scheduler._skip_zero_block_ids = {10, 12}
|
||||
|
||||
assert scheduler._get_new_block_ids_to_zero() == [9, 11]
|
||||
assert not scheduler._skip_zero_block_ids
|
||||
|
||||
|
||||
@pytest.mark.cpu_test
|
||||
def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
"""A failed async load leaves zeroing-skipped blocks unwritten beyond
|
||||
@@ -1575,7 +1547,6 @@ def test_failed_load_rezeroes_unwritten_skipped_blocks():
|
||||
|
||||
# Attention blocks covering tokens >= 48 are re-recorded for zeroing
|
||||
# and flow into the next step's zero list; Mamba blocks are not.
|
||||
scheduler._skip_zero_block_ids = set()
|
||||
assert scheduler._get_new_block_ids_to_zero() == [13, 14, 15]
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user