[Bugfix][Core] Exempt exactly the blocks an async KV load writes from zeroing (#59504)

Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Codex <noreply@openai.com>
This commit is contained in:
Yifan Qiao
2026-10-03 06:26:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Codex
parent 1a001d5842
commit ca0df1f122
17 changed files with 162 additions and 72 deletions
+52
View File
@@ -4,6 +4,8 @@
the hit to rebuild the non-cacheable sliding-window group, keeps the hit's
blocks, and hands the worker the replay start."""
import dataclasses
import pytest
import torch
@@ -277,3 +279,53 @@ def test_hit_no_longer_than_window_is_ignored(via_connector):
assert new_req.num_computed_tokens == 0
assert new_req.replay_start == 0
assert out.num_scheduled_tokens[request.request_id] == NUM_PROMPT_TOKENS
@pytest.mark.parametrize("is_async", [True, False])
@pytest.mark.parametrize("loads_window", [True, False])
def test_kv_load_zeroing_respects_loaded_groups(is_async, loads_window):
"""Only groups restored by an async load skip zeroing. P/D loads include
the non-cacheable window; stores leave it to be initialized locally."""
local = 5 * BLOCK_SIZE
scheduler = _replay_scheduler(use_kv_connector=MockKVConfig())
if loads_window:
scheduler.connector.get_loaded_kv_cache_group_ids = lambda request: (FULL, SWA)
assert scheduler.needs_kv_cache_zeroing
first = create_requests(
num_requests=1, num_tokens=local, same_prompt=True, block_size=BLOCK_SIZE
)[0]
_prefill(scheduler, first)
scheduler.connector.config = dataclasses.replace(
scheduler.connector.config,
matched_tokens=NUM_PROMPT_TOKENS - 1 - local,
is_async=is_async,
)
request = create_requests(
num_requests=1,
num_tokens=NUM_PROMPT_TOKENS,
same_prompt=True,
block_size=BLOCK_SIZE,
req_ids=["load"],
)[0]
scheduler.add_request(request)
out = scheduler.schedule()
manager = scheduler.kv_cache_manager
hit_ids = {
b.block_id for g in manager.get_blocks(first.request_id).blocks for b in g
}
swa_blocks = manager.get_blocks(request.request_id).blocks[SWA]
assert any(
not b.is_null and b.block_id not in hit_ids
for b in swa_blocks[: local // BLOCK_SIZE]
)
zeroed = set(out.new_block_ids_to_zero or ())
for group_id, blocks in enumerate(manager.get_blocks(request.request_id).blocks):
new_ids = {
b.block_id for b in blocks if not b.is_null and b.block_id not in hit_ids
}
assert new_ids
if is_async and (group_id == FULL or loads_window):
assert not new_ids & zeroed
else:
assert new_ids <= zeroed
@@ -16,6 +16,7 @@ from vllm.v1.core.single_type_kv_cache_manager import (
ChunkedLocalAttentionManager,
CircularBufferManager,
FullAttentionManager,
KpoolTailManager,
MambaManager,
RSWAManager,
SlidingWindowManager,
@@ -24,6 +25,7 @@ from vllm.v1.kv_cache_interface import (
ChunkedLocalAttentionSpec,
CircularBufferSpec,
FullAttentionSpec,
KpoolTailSpec,
MambaSpec,
RSWASpec,
SlidingWindowSpec,
@@ -332,6 +334,37 @@ def test_circular_buffer_allocates_one_block_for_the_request_lifetime():
assert manager.req_to_blocks[request_id] == blocks
@pytest.mark.parametrize("record_for_zeroing", [True, False])
def test_external_kpool_tail_zeroing(record_for_zeroing):
"""A transferred ring at index zero skips zeroing even with a local hit."""
spec = KpoolTailSpec(
block_size=4,
num_kv_heads=2,
head_size=8,
dtype=torch.bfloat16,
sliding_window=4,
)
manager = KpoolTailManager(
spec,
block_pool=BlockPool(10, enable_caching=True, hash_block_size=4),
enable_caching=True,
kv_cache_group_id=0,
scheduler_block_size=4,
needs_kv_cache_zeroing=True,
max_admission_blocks_per_request=1,
)
manager.allocate_external_computed_blocks(
"request", 16, 4, record_for_zeroing=record_for_zeroing
)
blocks = manager.req_to_blocks["request"]
assert len(blocks) == 1
assert manager.take_new_block_ids() == (
[blocks[0].block_id] if record_for_zeroing else []
)
assert manager.allocate_new_blocks("request", 20, 20) == []
assert manager.take_new_block_ids() == []
def test_sliding_window_records_new_blocks_for_zeroing():
block_size = 2
spec = SlidingWindowSpec(