Signed-off-by: Saddss <2872669061@qq.com>
This commit is contained in:
@@ -22,7 +22,7 @@ from vllm.multimodal.inputs import (
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.hashing import sha256, sha256_cbor
|
||||
from vllm.v1.core.block_pool import BlockHashToBlockMap, BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheManager, Request
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks, KVCacheManager, Request
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
BlockHash,
|
||||
BlockHashWithGroupId,
|
||||
@@ -3519,6 +3519,180 @@ def test_can_fit_full_sequence_full_attention_still_gates_oversized():
|
||||
assert manager.allocate_slots(req, block_size, full_sequence_must_fit=True) is None
|
||||
|
||||
|
||||
def test_cache_hit_local_and_external():
|
||||
# Regression test for #33775: when a request hits the local prefix cache
|
||||
# in one KV cache group and needs external (connector) blocks in another,
|
||||
# the external allocation of an earlier group must not evict the local
|
||||
# cache-hit blocks of a later group. Otherwise the same physical block can
|
||||
# be handed out twice, producing duplicate block IDs / ref_cnt corruption.
|
||||
block_size = 16
|
||||
kv_cache_config = make_kv_cache_config_hybrid_model(block_size, 31, 100)
|
||||
del kv_cache_config.kv_cache_groups[2:]
|
||||
req_id = "test"
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
top_blocks = []
|
||||
head = manager.block_pool.free_block_queue.fake_free_list_head
|
||||
for _ in range(10):
|
||||
top_blocks.append(head.next_free_block)
|
||||
head = head.next_free_block
|
||||
cache_hit = KVCacheBlocks((top_blocks[:5], top_blocks[5:]))
|
||||
|
||||
manager.allocate_slots(
|
||||
make_request(req_id, [0] * (8 * block_size), block_size, sha256),
|
||||
16,
|
||||
5 * block_size,
|
||||
cache_hit,
|
||||
0,
|
||||
2 * block_size,
|
||||
)
|
||||
|
||||
req_blocks = manager.get_blocks(req_id)
|
||||
req_block_ids = req_blocks.get_block_ids()
|
||||
all_block_ids = req_block_ids[0] + req_block_ids[1]
|
||||
assert len(set(all_block_ids)) == len(all_block_ids), "Block IDs are not unique"
|
||||
|
||||
|
||||
def _take_free_blocks(manager: KVCacheManager, num_blocks: int) -> list[KVCacheBlock]:
|
||||
"""Grab the first ``num_blocks`` blocks at the head of the free queue
|
||||
without removing them. These ref_cnt==0 blocks stand in for evictable
|
||||
cache-hit blocks left behind by a previous (e.g. preempted) request, and
|
||||
sitting at the head guarantees a later group's external ``get_new_blocks``
|
||||
would contend for them on unpatched code (issue #33775)."""
|
||||
blocks: list[KVCacheBlock] = []
|
||||
head = manager.block_pool.free_block_queue.fake_free_list_head
|
||||
for _ in range(num_blocks):
|
||||
head = head.next_free_block
|
||||
blocks.append(head)
|
||||
return blocks
|
||||
|
||||
|
||||
def _assert_no_double_allocation(manager: KVCacheManager, req_id: str) -> None:
|
||||
"""No physical block may be handed out twice across groups, and every
|
||||
block referenced by the request must have a live ref_cnt."""
|
||||
block_ids = manager.get_blocks(req_id).get_block_ids()
|
||||
flat = [block_id for group in block_ids for block_id in group]
|
||||
assert len(set(flat)) == len(flat), "Block IDs are not unique across groups"
|
||||
null_id = manager.block_pool.null_block.block_id
|
||||
for block_id in flat:
|
||||
if block_id == null_id:
|
||||
continue
|
||||
assert manager.block_pool.blocks[block_id].ref_cnt >= 1, (
|
||||
f"block {block_id} referenced by the request has ref_cnt 0"
|
||||
)
|
||||
|
||||
|
||||
def _two_phase_block_size(manager: KVCacheManager) -> int:
|
||||
return manager.kv_cache_config.kv_cache_groups[0].kv_cache_spec.block_size
|
||||
|
||||
|
||||
def _cross_group_cache_hit(
|
||||
manager: KVCacheManager,
|
||||
req_id: str,
|
||||
num_groups: int,
|
||||
local_blocks_per_group: int = 5,
|
||||
num_external_blocks: int = 2,
|
||||
num_new_blocks: int = 1,
|
||||
) -> Request:
|
||||
"""Allocate ``req_id`` with a per-group local prefix hit plus external
|
||||
(connector) computed tokens, driving the coordinator's two-phase path.
|
||||
Returns the allocated request so callers can free it (e.g. to preempt)."""
|
||||
block_size = _two_phase_block_size(manager)
|
||||
hit_blocks = _take_free_blocks(manager, num_groups * local_blocks_per_group)
|
||||
cache_hit = KVCacheBlocks(
|
||||
tuple(
|
||||
hit_blocks[i * local_blocks_per_group : (i + 1) * local_blocks_per_group]
|
||||
for i in range(num_groups)
|
||||
)
|
||||
)
|
||||
prompt_blocks = local_blocks_per_group + num_external_blocks + num_new_blocks
|
||||
request = make_request(
|
||||
req_id, [0] * (prompt_blocks * block_size), block_size, sha256
|
||||
)
|
||||
manager.allocate_slots(
|
||||
request,
|
||||
num_new_blocks * block_size,
|
||||
local_blocks_per_group * block_size,
|
||||
cache_hit,
|
||||
0,
|
||||
num_external_blocks * block_size,
|
||||
)
|
||||
return request
|
||||
|
||||
|
||||
def _make_two_phase_manager(num_groups: int) -> KVCacheManager:
|
||||
assert num_groups in (2, 3)
|
||||
block_size = 16
|
||||
kv_cache_config = make_kv_cache_config_hybrid_model(block_size, 31, 100)
|
||||
del kv_cache_config.kv_cache_groups[num_groups:]
|
||||
return make_kv_cache_manager(
|
||||
kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=block_size,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
|
||||
def test_cache_hit_local_and_external_three_groups():
|
||||
# Scenario 1 (issue #33775): SWA + full attention with *three* KV cache
|
||||
# groups (1 full + 2 sliding-window). A local prefix hit in some groups
|
||||
# combined with external (connector) blocks in others must not let one
|
||||
# group's external `get_new_blocks` evict another group's not-yet-touched
|
||||
# cache-hit blocks, which would hand the same physical block out twice.
|
||||
manager = _make_two_phase_manager(num_groups=3)
|
||||
_cross_group_cache_hit(manager, "test", num_groups=3)
|
||||
_assert_no_double_allocation(manager, "test")
|
||||
|
||||
|
||||
def test_cache_hit_local_and_external_three_groups_preempt_and_reallocate():
|
||||
# Scenario 2: the same 3-group hybrid config, but the request is preempted
|
||||
# (freed) and then reallocated. After the free, the coordinator must treat
|
||||
# the request as new again so external blocks are re-allocated, and the
|
||||
# two-phase ordering must still prevent cross-group double allocation when
|
||||
# reallocating against the now-evictable cache-hit blocks.
|
||||
manager = _make_two_phase_manager(num_groups=3)
|
||||
|
||||
request = _cross_group_cache_hit(manager, "test", num_groups=3)
|
||||
_assert_no_double_allocation(manager, "test")
|
||||
|
||||
# Preempt: free the request; its blocks return to the pool (full ones stay
|
||||
# cached/evictable) and the coordinator forgets it.
|
||||
manager.free(request)
|
||||
assert manager.get_blocks("test").get_block_ids() == ([], [], [])
|
||||
|
||||
# Reallocate the same request id against fresh cache-hit blocks taken from
|
||||
# the current free-queue head, mirroring a preempted request being
|
||||
# scheduled again. Because the request is no longer known, the coordinator
|
||||
# re-arms `is_new_request` and re-runs external allocation, which must still
|
||||
# not double-allocate across groups.
|
||||
_cross_group_cache_hit(manager, "test", num_groups=3)
|
||||
_assert_no_double_allocation(manager, "test")
|
||||
assert manager.get_blocks("test").get_block_ids() != ([], [], [])
|
||||
|
||||
|
||||
def test_cache_hit_local_and_external_two_groups_preempt_and_reallocate():
|
||||
# Scenario 3: the minimal 2-group hybrid config (1 full + 1 sliding-window)
|
||||
# exercised through the same preempt -> reallocate cycle as scenario 2.
|
||||
manager = _make_two_phase_manager(num_groups=2)
|
||||
|
||||
request = _cross_group_cache_hit(manager, "test", num_groups=2)
|
||||
_assert_no_double_allocation(manager, "test")
|
||||
|
||||
manager.free(request)
|
||||
assert manager.get_blocks("test").get_block_ids() == ([], [])
|
||||
|
||||
_cross_group_cache_hit(manager, "test", num_groups=2)
|
||||
_assert_no_double_allocation(manager, "test")
|
||||
assert manager.get_blocks("test").get_block_ids() != ([], [])
|
||||
|
||||
|
||||
def test_swa_free_split_keeps_cached_tail_ahead_of_scratch(monkeypatch):
|
||||
"""Default path (no retention): freeing an SWA request must place its
|
||||
uncached scratch blocks at the front of the free queue (recycled first)
|
||||
|
||||
@@ -390,7 +390,7 @@ def test_evictable_cached_blocks_not_double_allocated():
|
||||
# should only allocate the truly new block.
|
||||
assert num_blocks_to_allocate == 2
|
||||
|
||||
manager.allocate_new_computed_blocks(
|
||||
manager.add_local_computed_blocks(
|
||||
request_id,
|
||||
[evictable_block],
|
||||
num_local_computed_tokens=block_size,
|
||||
|
||||
Reference in New Issue
Block a user