[Bugfix] Two-phase KV allocation for cross-group prefix cache hits (supersedes #33775) (#44409)

Signed-off-by: Saddss <2872669061@qq.com>
This commit is contained in:
Saddss
2026-06-15 22:39:59 +08:00
committed by GitHub
parent fa63bb9db6
commit 588db18362
4 changed files with 255 additions and 34 deletions
+175 -1
View File
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
from vllm.sampling_params import SamplingParams
from vllm.utils.hashing import sha256, sha256_cbor
from vllm.v1.core.block_pool import BlockHashToBlockMap, BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager, Request
from vllm.v1.core.kv_cache_utils import (
BlockHash,
BlockHashWithGroupId,
@@ -3519,6 +3519,180 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized():
assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None
def test_cache_hit_local_and_external():
# Regression test for #33775: when a request hits the local prefix cache
# in one KV cache group and needs external (connector) blocks in another,
# the external allocation of an earlier group must not evict the local
# cache-hit blocks of a later group. Otherwise the same physical block can
# be handed out twice, producing duplicate block IDs / ref_cnt corruption.
block_size = 16
kv_cache_config = make_kv_cache_config_hybrid_model(block_size, 31, 100)
del kv_cache_config.kv_cache_groups[2:]
req_id = "test"
manager = make_kv_cache_manager(
kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
use_eagle=True,
)
top_blocks = []
head = manager.block_pool.free_block_queue.fake_free_list_head
for _ in range(10):
top_blocks.append(head.next_free_block)
head = head.next_free_block
cache_hit = KVCacheBlocks((top_blocks[:5], top_blocks[5:]))
manager.allocate_slots(
make_request(req_id, [0] * (8 * block_size), block_size, sha256),
16,
5 * block_size,
cache_hit,
0,
2 * block_size,
)
req_blocks = manager.get_blocks(req_id)
req_block_ids = req_blocks.get_block_ids()
all_block_ids = req_block_ids[0] + req_block_ids[1]
assert len(set(all_block_ids)) == len(all_block_ids), "Block IDs are not unique"
def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheBlock]:
"""Grab the first ``num_blocks`` blocks at the head of the free queue
without removing them. These ref_cnt==0 blocks stand in for evictable
cache-hit blocks left behind by a previous (e.g. preempted) request, and
sitting at the head guarantees a later group's external ``get_new_blocks``
would contend for them on unpatched code (issue #33775)."""
blocks: list[KVCacheBlock] = []
head = manager.block_pool.free_block_queue.fake_free_list_head
for _ in range(num_blocks):
head = head.next_free_block
blocks.append(head)
return blocks
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
"""No physical block may be handed out twice across groups, and every
block referenced by the request must have a live ref_cnt."""
block_ids = manager.get_blocks(req_id).get_block_ids()
flat = [block_id for group in block_ids for block_id in group]
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
null_id = manager.block_pool.null_block.block_id
for block_id in flat:
if block_id == null_id:
continue
assert manager.block_pool.blocks[block_id].ref_cnt >= 1, (
f"block {block_id} referenced by the request has ref_cnt 0"
)
def _two_phase_block_size(manager: KVCacheManager) -> int:
return manager.kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
def _cross_group_cache_hit(
manager: KVCacheManager,
req_id: str,
num_groups: int,
local_blocks_per_group: int = 5,
num_external_blocks: int = 2,
num_new_blocks: int = 1,
) -> Request:
"""Allocate ``req_id`` with a per-group local prefix hit plus external
(connector) computed tokens, driving the coordinator's two-phase path.
Returns the allocated request so callers can free it (e.g. to preempt)."""
block_size = _two_phase_block_size(manager)
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
cache_hit = KVCacheBlocks(
tuple(
hit_blocks[i * local_blocks_per_group : (i + 1) * local_blocks_per_group]
for i in range(num_groups)
)
)
prompt_blocks = local_blocks_per_group + num_external_blocks + num_new_blocks
request = make_request(
req_id, [0] * (prompt_blocks * block_size), block_size, sha256
)
manager.allocate_slots(
request,
num_new_blocks * block_size,
local_blocks_per_group * block_size,
cache_hit,
0,
num_external_blocks * block_size,
)
return request
def _make_two_phase_manager(num_groups: int) -> KVCacheManager:
assert num_groups in (2, 3)
block_size = 16
kv_cache_config = make_kv_cache_config_hybrid_model(block_size, 31, 100)
del kv_cache_config.kv_cache_groups[num_groups:]
return make_kv_cache_manager(
kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=block_size,
use_eagle=True,
)
def test_cache_hit_local_and_external_three_groups():
# Scenario 1 (issue #33775): SWA + full attention with *three* KV cache
# groups (1 full + 2 sliding-window). A local prefix hit in some groups
# combined with external (connector) blocks in others must not let one
# group's external `get_new_blocks` evict another group's not-yet-touched
# cache-hit blocks, which would hand the same physical block out twice.
manager = _make_two_phase_manager(num_groups=3)
_cross_group_cache_hit(manager, "test", num_groups=3)
_assert_no_double_allocation(manager, "test")
def test_cache_hit_local_and_external_three_groups_preempt_and_reallocate():
# Scenario 2: the same 3-group hybrid config, but the request is preempted
# (freed) and then reallocated. After the free, the coordinator must treat
# the request as new again so external blocks are re-allocated, and the
# two-phase ordering must still prevent cross-group double allocation when
# reallocating against the now-evictable cache-hit blocks.
manager = _make_two_phase_manager(num_groups=3)
request = _cross_group_cache_hit(manager, "test", num_groups=3)
_assert_no_double_allocation(manager, "test")
# Preempt: free the request; its blocks return to the pool (full ones stay
# cached/evictable) and the coordinator forgets it.
manager.free(request)
assert manager.get_blocks("test").get_block_ids() == ([], [], [])
# Reallocate the same request id against fresh cache-hit blocks taken from
# the current free-queue head, mirroring a preempted request being
# scheduled again. Because the request is no longer known, the coordinator
# re-arms `is_new_request` and re-runs external allocation, which must still
# not double-allocate across groups.
_cross_group_cache_hit(manager, "test", num_groups=3)
_assert_no_double_allocation(manager, "test")
assert manager.get_blocks("test").get_block_ids() != ([], [], [])
def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():
# Scenario 3: the minimal 2-group hybrid config (1 full + 1 sliding-window)
# exercised through the same preempt -> reallocate cycle as scenario 2.
manager = _make_two_phase_manager(num_groups=2)
request = _cross_group_cache_hit(manager, "test", num_groups=2)
_assert_no_double_allocation(manager, "test")
manager.free(request)
assert manager.get_blocks("test").get_block_ids() == ([], [])
_cross_group_cache_hit(manager, "test", num_groups=2)
_assert_no_double_allocation(manager, "test")
assert manager.get_blocks("test").get_block_ids() != ([], [])
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
"""Default path (no retention): freeing an SWA request must place its
uncached scratch blocks at the front of the free queue (recycled first)
@@ -390,7 +390,7 @@ def test_evictable_cached_blocks_not_double_allocated():
# should only allocate the truly new block.
assert num_blocks_to_allocate == 2
manager.allocate_new_computed_blocks(
manager.add_local_computed_blocks(
request_id,
[evictable_block],
num_local_computed_tokens=block_size,