[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
@@ -0,0 +1,816 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
"align") models: scheduler chunk splitting, partial tail registration, CoW
on partial hits, and same-step deferral."""
from types import SimpleNamespace
import pytest
import torch
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
from vllm.utils.hashing import sha256
from vllm.v1.core.kv_cache_utils import (
KVCacheBlockCopy,
get_block_hash,
get_group_id,
init_none_hash,
)
from vllm.v1.core.sched.scheduler import Scheduler
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheConfig,
KVCacheGroupSpec,
MambaSpec,
)
@pytest.fixture(autouse=True)
def _auto_init_hash_fn():
init_none_hash(sha256)
def test_mamba_align_split_partial_tail_schedule():
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
at the prompt's last hash boundary (registering the partial tail), then
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
0 -> 8192 -> 9728 -> 9984 -> 10000."""
block_size = 512
hash_block_size = 32
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size),
use_eagle=False,
hash_block_size=hash_block_size,
mamba_partial_cache_hit=True,
)
split = Scheduler._mamba_block_aligned_split
req = make_request("0", [0] * 10000, hash_block_size, sha256)
req.num_computed_tokens = 0
assert split(self=mock, request=req, num_new_tokens=8192) == 8192
req.num_computed_tokens = 8192
# Stop at the last block boundary (9728).
assert split(self=mock, request=req, num_new_tokens=1808) == 1536
req.num_computed_tokens = 9728
# Extra stop at the prompt's last hash boundary (9984).
assert split(self=mock, request=req, num_new_tokens=272) == 256
req.num_computed_tokens = 9984
# Final 16 tokens run unchanged (no mid-block-resume stop: the next
# block boundary is past the last block boundary).
assert split(self=mock, request=req, num_new_tokens=16) == 16
# Partial hits off: no extra stop, the tail runs in one chunk.
mock.mamba_partial_cache_hit = False
req.num_computed_tokens = 9728
assert split(self=mock, request=req, num_new_tokens=272) == 272
mock.mamba_partial_cache_hit = True
# A request resumed mid-block (partial hash hit at 9984): the first chunk
# stops at the next block boundary (10240), later chunk ends re-align.
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
req2.num_computed_tokens = 9984
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
req2.num_computed_tokens = 10240
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
def test_hybrid_mamba_align_partial_hash_hit():
hash_block_size = 2
mamba_block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=20,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=mamba_block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert num_computed == 0
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
assert blocks is not None
manager.free(req0)
manager.new_step_starts()
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_mamba_block = manager.block_pool.get_cached_block(
partial_mamba_hash, kv_cache_group_ids=[1]
)
assert partial_mamba_block is not None
assert partial_mamba_block[0].block_hash_num_tokens == 6
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 6
assert [len(group) for group in computed_blocks.blocks] == [3, 2]
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
assert new_blocks is not None
mamba_new_block_ids = new_blocks.get_block_ids()[1]
assert len(mamba_new_block_ids) == 1
assert mamba_new_block_ids[0] != partial_mamba_block[0].block_id
assert manager.get_blocks("1").get_block_ids()[1][1] == mamba_new_block_ids[0]
assert partial_mamba_block[0].block_hash is not None
assert get_block_hash(partial_mamba_block[0].block_hash) == partial_mamba_hash
assert get_group_id(partial_mamba_block[0].block_hash) == 1
assert partial_mamba_block[0].block_hash_num_tokens == 6
copies, _ = manager.take_kv_cache_block_copies()
assert (
KVCacheBlockCopy(
src_block_id=partial_mamba_block[0].block_id,
dst_block_id=mamba_new_block_ids[0],
)
in copies
)
assert manager.get_blocks("1").blocks[1][1].block_hash_num_tokens == 8
def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_mamba_block = manager.block_pool.get_cached_block(
partial_mamba_hash, kv_cache_group_ids=[1]
)
assert partial_mamba_block is not None
partial_mamba_block_id = partial_mamba_block[0].block_id
assert manager.get_blocks("0").get_block_ids()[1][1] == partial_mamba_block_id
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
new_blocks = manager.allocate_slots(req0, 1)
assert new_blocks is not None
# Reversed CoW for the owning request: it keeps its own block (the
# worker's block table is append-only), and no new mamba block is handed
# to the worker. The prefix-cache entry is moved to a private copy that
# the queued block copy fills before the next forward.
assert new_blocks.get_block_ids()[1] == []
assert manager.get_blocks("0").get_block_ids()[1][1] == partial_mamba_block_id
copies, _ = manager.take_kv_cache_block_copies()
cow_copy = next(c for c in copies if c.src_block_id == partial_mamba_block_id)
assert cow_copy.dst_block_id != partial_mamba_block_id
# The source block gave up the hash; the copy target now owns the entry.
assert partial_mamba_block[0].block_hash is None
moved = manager.block_pool.get_cached_block(
partial_mamba_hash, kv_cache_group_ids=[1]
)
assert moved is not None
assert moved[0].block_id == cow_copy.dst_block_id
assert get_block_hash(moved[0].block_hash) == partial_mamba_hash
assert get_group_id(moved[0].block_hash) == 1
assert moved[0].block_hash_num_tokens == 6
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_mamba_block = manager.block_pool.get_cached_block(
partial_mamba_hash, kv_cache_group_ids=[1]
)
assert partial_mamba_block is not None
partial_mamba_block_id = partial_mamba_block[0].block_id
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# The owner moved the prefix-cache entry to a private copy; capture its id.
owner_copies, _ = manager.take_kv_cache_block_copies()
cow_copy = next(c for c in owner_copies if c.src_block_id == partial_mamba_block_id)
moved_block_id = cow_copy.dst_block_id
manager.new_step_starts()
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 6
# The later request hits the moved (private-copy) entry, not the source.
assert computed_blocks.get_block_ids()[1][1] == moved_block_id
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
assert new_blocks is not None
mamba_new_block_ids = new_blocks.get_block_ids()[1]
assert len(mamba_new_block_ids) == 1
assert mamba_new_block_ids[0] != moved_block_id
# The hitting request CoWs from the moved entry into its own private block.
copies, _ = manager.take_kv_cache_block_copies()
assert (
KVCacheBlockCopy(
src_block_id=moved_block_id,
dst_block_id=mamba_new_block_ids[0],
)
in copies
)
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
"""The owner's move re-arms the same-step guard: the moved entry is
filled by this step's copy, and chained same-step copies read stale
sources, so a request hitting it in the move step must be deferred."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.new_step_starts()
# The owning request continues decoding: the partial entry moves to a
# private copy in this step.
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
# A request hitting the moved entry in the SAME step must be deferred.
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 6
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is None
# Next step the moved entry is consumable.
manager.new_step_starts()
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 6
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is not None
def test_hybrid_full_attention_partial_hash_hit_uses_cow():
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
manager.new_step_starts()
partial_full_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_full_block = manager.block_pool.get_cached_block(
partial_full_hash, kv_cache_group_ids=[0]
)
assert partial_full_block is not None
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 6
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
assert new_blocks is not None
full_new_block_ids = new_blocks.get_block_ids()[0]
assert len(full_new_block_ids) == 1
assert full_new_block_ids[0] != partial_full_block[0].block_id
assert partial_full_block[0].block_hash is not None
assert get_block_hash(partial_full_block[0].block_hash) == partial_full_hash
assert get_group_id(partial_full_block[0].block_hash) == 0
assert partial_full_block[0].block_hash_num_tokens == 6
copies, retained = manager.take_kv_cache_block_copies()
assert (
KVCacheBlockCopy(
src_block_id=partial_full_block[0].block_id,
dst_block_id=full_new_block_ids[0],
)
in copies
)
assert partial_full_block[0].ref_cnt == 1
manager.block_pool.free_blocks(retained)
assert partial_full_block[0].ref_cnt == 0
def test_hybrid_partial_hit_cow_target_starts_uncached():
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert num_computed == 0
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
manager.free(req0)
manager.new_step_starts()
partial_hash = req0.block_hashes[6 // hash_block_size - 1]
partial_full_block = manager.block_pool.get_cached_block(
partial_hash, kv_cache_group_ids=[0]
)
partial_mamba_block = manager.block_pool.get_cached_block(
partial_hash, kv_cache_group_ids=[1]
)
assert partial_full_block is not None
assert partial_mamba_block is not None
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 6
new_blocks = manager.allocate_slots(
req1,
2,
num_computed,
computed_blocks,
delay_cache_blocks=True,
)
assert new_blocks is not None
full_cow_block = manager.get_blocks("1").blocks[0][1]
mamba_cow_block = manager.get_blocks("1").blocks[1][1]
assert full_cow_block.block_id != partial_full_block[0].block_id
assert mamba_cow_block.block_id != partial_mamba_block[0].block_id
assert full_cow_block.block_hash is None
assert full_cow_block.block_hash_num_tokens is None
assert mamba_cow_block.block_hash is None
assert mamba_cow_block.block_hash_num_tokens is None
assert partial_full_block[0].block_hash is not None
assert get_block_hash(partial_full_block[0].block_hash) == partial_hash
assert get_group_id(partial_full_block[0].block_hash) == 0
assert partial_full_block[0].block_hash_num_tokens == 6
assert partial_mamba_block[0].block_hash is not None
assert get_block_hash(partial_mamba_block[0].block_hash) == partial_hash
assert get_group_id(partial_mamba_block[0].block_hash) == 1
assert partial_mamba_block[0].block_hash_num_tokens == 6
def test_hybrid_partial_hash_truncates_full_attention_hit_length():
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
pool = manager.block_pool
req = make_request(
"0",
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
hash_block_size,
sha256,
)
full_blocks = pool.get_new_blocks(3)
pool.cache_full_blocks(
request=req,
blocks=full_blocks,
num_cached_blocks=0,
num_full_blocks=2,
block_size=block_size,
kv_cache_group_id=0,
)
pool.cache_partial_block(
request=req,
block=full_blocks[2],
num_tokens=10,
kv_cache_group_id=0,
block_size=block_size,
)
mamba_block = pool.get_new_blocks(1)[0]
pool.cache_partial_block(
request=req,
block=mamba_block,
num_tokens=6,
kv_cache_group_id=1,
block_size=block_size,
)
computed_blocks, num_computed = manager.get_computed_blocks(req)
assert num_computed == 6
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
def test_cow_retained_blocks_returned_for_release():
"""new_step_starts returns the CoW copy retentions instead of freeing
them; the scheduler owns releasing them once the copy has run."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=24,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=hash_block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
)
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
# The owner's move queues a copy and retains both endpoints.
req0.num_computed_tokens = 6
req0.append_output_token_ids([3])
assert manager.allocate_slots(req0, 1) is not None
(cow_copy,), retained = manager.take_kv_cache_block_copies()
assert {b.block_id for b in retained} == {
cow_copy.src_block_id,
cow_copy.dst_block_id,
}
# Not freed yet: the retention refs are still held.
assert all(b.ref_cnt > 0 for b in retained)
manager.block_pool.free_blocks(retained)
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
"""Scheduler releases CoW retentions immediately when the copy's step has
been processed (or deferral is off), and defers them otherwise."""
from collections import deque
freed: list = []
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=9)]
mock = SimpleNamespace(
kv_cache_manager=SimpleNamespace(
block_pool=SimpleNamespace(free_blocks=freed.extend)
),
deferred_frees=deque(),
defer_block_free=True,
processed_step_seq=2,
)
free = Scheduler._free_cow_retained_blocks
# Copy step still in flight: deferred with its fence.
free(mock, list(blocks), fence_seq=3)
assert not freed
assert mock.deferred_frees == deque([(3, blocks[::-1])])
# Copy step processed: freed immediately.
mock.processed_step_seq = 3
free(mock, list(blocks), fence_seq=3)
assert freed == blocks
# Deferral disabled: freed immediately regardless of the fence.
freed.clear()
mock.deferred_frees.clear()
mock.defer_block_free = False
mock.processed_step_seq = 0
free(mock, list(blocks), fence_seq=3)
assert freed == blocks
def test_full_attention_eagle_drops_one_hash_unit():
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
instead of a whole cache block: the tail block's KV is append-only, so it
still covers the reduced length and stays in the hit as a partial block."""
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
hash_block_size = 2
block_size = 4
pool = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=hash_block_size
)
spec = FullAttentionSpec(
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
)
req = make_request("0", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
def find(drop_eagle_block):
return FullAttentionManager.find_longest_cache_hit(
block_hashes=req.block_hashes,
max_length=8,
kv_cache_group_ids=[0],
block_pool=pool,
kv_cache_spec=spec,
drop_eagle_block=drop_eagle_block,
alignment_tokens=hash_block_size,
)
# Two full cached blocks (hit 8): eagle rewinds to 6, keeping the last
# block as a partial hit instead of dropping it to 4.
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks,
num_cached_blocks=0,
num_full_blocks=2,
block_size=block_size,
kv_cache_group_id=0,
)
hit_blocks, hit_length = find(drop_eagle_block=False)
assert (hit_length, len(hit_blocks[0])) == (8, 2)
hit_blocks, hit_length = find(drop_eagle_block=True)
assert (hit_length, len(hit_blocks[0])) == (6, 2)
# A partial tail at 6 (block 1 not fully cached): eagle rewinds to the
# block boundary and trims the tail block.
pool2 = BlockPool(
num_gpu_blocks=10, enable_caching=True, hash_block_size=hash_block_size
)
pool = pool2
blocks = pool.get_new_blocks(2)
pool.cache_full_blocks(
request=req,
blocks=blocks[:1],
num_cached_blocks=0,
num_full_blocks=1,
block_size=block_size,
kv_cache_group_id=0,
)
assert (
pool.cache_partial_block(
request=req,
block=blocks[1],
num_tokens=6,
kv_cache_group_id=0,
block_size=block_size,
)
is not None
)
hit_blocks, hit_length = find(drop_eagle_block=False)
assert (hit_length, len(hit_blocks[0])) == (6, 2)
hit_blocks, hit_length = find(drop_eagle_block=True)
assert (hit_length, len(hit_blocks[0])) == (4, 1)
def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
"""Regression: with eagle, the mamba group must not receive the eagle
lookup margin — its finder never applies the drop, so it could return a
hit past the blocks the (dropped) full-attention group covers, crashing
the consumer's CoW with block_idx >= len(req_blocks)."""
hash_block_size = 2
block_size = 2 * hash_block_size
kv_cache_config = KVCacheConfig(
num_blocks=32,
kv_cache_tensors=[],
kv_cache_groups=[
KVCacheGroupSpec(
["full"],
FullAttentionSpec(
block_size=block_size,
num_kv_heads=1,
head_size=1,
dtype=torch.float32,
),
),
KVCacheGroupSpec(
["mamba"],
MambaSpec(
block_size=block_size,
shapes=(1, 1),
dtypes=(torch.float32,),
mamba_cache_mode="align",
),
),
],
)
manager = make_kv_cache_manager(
kv_cache_config=kv_cache_config,
max_model_len=8192,
enable_caching=True,
hash_block_size=hash_block_size,
use_eagle=True,
)
# The owner prefills in scheduler-split style: stop at the block boundary
# (4), then at the prompt's last hash boundary (6, partial entries).
req0 = make_request("0", [7] * 6, hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req0)
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
req0.num_computed_tokens = 4
manager.new_step_starts()
assert manager.allocate_slots(req0, 2) is not None
req0.num_computed_tokens = 6
manager.new_step_starts()
# A longer request with eagle: full attention drops the partial tail, so
# the joint hit must fall back to the block boundary the FA blocks cover.
req1 = make_request("1", [7] * 6 + [9] * 2, hash_block_size, sha256)
computed_blocks, num_computed = manager.get_computed_blocks(req1)
assert num_computed == 4
assert all(
len(group) * block_size >= num_computed for group in computed_blocks.blocks
)
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
+62
View File
@@ -18,6 +18,7 @@ from unittest.mock import PropertyMock, patch
import pytest
from vllm.config import VllmConfig
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.outputs import ModelRunnerOutput
from vllm.v1.request import RequestStatus
@@ -412,3 +413,64 @@ def test_non_async_abort_defers_via_last_sched_seq():
scheduler.update_from_output(out0, _make_model_runner_output(out0))
assert not scheduler.deferred_frees
assert pool.get_num_free_blocks() == num_free_initially
def test_cow_retentions_deferred_until_copy_step_processed():
"""The endpoints of a queued KV block copy must stay out of the free
pool until the step that runs the copy has been processed. Freed
earlier, an endpoint can be reallocated (e.g. as a PD KV-load
destination) and overwritten by a transfer that is not ordered against
the copy still pending in the in-flight step.
"""
scheduler = _create_deferring_scheduler()
pool = scheduler.kv_cache_manager.block_pool
manager = scheduler.kv_cache_manager.coordinator.single_type_managers[0]
request = create_requests(
num_requests=1,
num_tokens=NUM_PROMPT_TOKENS,
max_tokens=5,
stop_token_ids=[STOP_TOKEN_ID],
)[0]
scheduler.add_request(request)
# Simulate a partial-hit CoW performed while scheduling step 1, whose
# hitting request was freed within the same step: the copy rides out0
# and each endpoint stays alive only through its copy retention.
src_block, dst_block = pool.get_new_blocks(2)
block_copy = KVCacheBlockCopy(
src_block_id=src_block.block_id, dst_block_id=dst_block.block_id
)
manager._pending_cow_copies.append((src_block, dst_block))
out0 = scheduler.schedule()
assert out0.kv_cache_block_copies == [block_copy]
# Exhaust the rest of the pool so the copy endpoints are the only blocks
# a new request could receive, then add one that fits exactly in them.
pool.get_new_blocks(pool.get_num_free_blocks())
late_request = create_requests(
num_requests=1,
num_tokens=2 * scheduler.block_size,
max_tokens=5,
req_ids=["late"],
)[0]
scheduler.add_request(late_request)
# Step 2 is scheduled while step 1 (which runs the copy) is still in
# flight: the retentions are released against the copy's fence, so the
# endpoints must not reach the free pool -- the late request must not be
# scheduled onto them.
out1 = scheduler.schedule()
assert src_block.ref_cnt == 1
assert dst_block.ref_cnt == 1
assert scheduler.deferred_frees
assert not out1.scheduled_new_reqs
# Step 1's output is processed: the copy has run, endpoints return to
# the pool and the late request can be scheduled onto them safely.
scheduler.update_from_output(out0, _make_model_runner_output(out0))
assert src_block.ref_cnt == 0
assert dst_block.ref_cnt == 0
assert not scheduler.deferred_frees
out2 = scheduler.schedule()
assert [r.req_id for r in out2.scheduled_new_reqs] == ["late"]
+60
View File
@@ -2597,3 +2597,63 @@ def test_hma_not_disabled_when_kv_events_enabled():
assert vllm_config.scheduler_config.disable_hybrid_kv_cache_manager is False, (
"kv_events_config must not force-disable the hybrid KV cache manager."
)
def test_resolve_block_hashes_gate():
# Resolve symbols through the module so they stay consistent with each other
# even after other tests reload ``kv_cache_utils``.
resolve_block_hashes = kv_cache_utils.resolve_block_hashes
BlockHashListWithBlockSize = kv_cache_utils.BlockHashListWithBlockSize
# Raw, hash_block_size-granularity hashes (contents are opaque here).
raw = [BlockHash(bytes([i])) for i in range(8)]
# block_size == hash_block_size: always reuse the raw hashes.
assert resolve_block_hashes(raw, 2, 2, alignment_tokens=2) is raw
assert (
resolve_block_hashes(
raw, 2, 2, supports_fine_grained_hash_lookup=True, alignment_tokens=2
)
is raw
)
# Fine-grained manager, partial hits ON (alignment_tokens == hash_block_size
# < block_size): keep raw hashes so the manager can scan at hash granularity.
assert (
resolve_block_hashes(
raw, 2, 4, supports_fine_grained_hash_lookup=True, alignment_tokens=2
)
is raw
)
# Fine-grained manager, partial hits OFF (alignment_tokens ==
# scheduler_block_size >= block_size): must fall back to a block-size view,
# exactly like the pre-refactor coordinator did.
off = resolve_block_hashes(
raw, 2, 4, supports_fine_grained_hash_lookup=True, alignment_tokens=4
)
assert isinstance(off, BlockHashListWithBlockSize)
assert off.scale_factor == 2
# Non-fine-grained manager (e.g. sliding window): always a block-size view
# when block_size != hash_block_size, regardless of alignment_tokens.
swa = resolve_block_hashes(
raw, 2, 4, supports_fine_grained_hash_lookup=False, alignment_tokens=2
)
assert isinstance(swa, BlockHashListWithBlockSize)
assert swa.scale_factor == 2
def test_resolve_block_hashes_rejects_mismatched_view():
resolve_block_hashes = kv_cache_utils.resolve_block_hashes
BlockHashListWithBlockSize = kv_cache_utils.BlockHashListWithBlockSize
raw = [BlockHash(bytes([i])) for i in range(8)]
# A view built at this block_size is returned as-is (idempotent).
view = BlockHashListWithBlockSize(raw, 2, 4)
assert resolve_block_hashes(view, 2, 4) is view
# A view built at a different block_size must fail loudly rather than be
# silently reinterpreted at the wrong granularity.
mismatched = BlockHashListWithBlockSize(raw, 2, 8)
with pytest.raises(AssertionError):
resolve_block_hashes(mismatched, 2, 4)
+4 -1
View File
@@ -1072,7 +1072,10 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
# Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0
# Create minimal mock with just the needed attributes
mock = SimpleNamespace(
cache_config=SimpleNamespace(block_size=block_size), use_eagle=False
cache_config=SimpleNamespace(block_size=block_size),
use_eagle=False,
hash_block_size=block_size,
mamba_partial_cache_hit=False,
)
num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split(
self=mock,
@@ -93,7 +93,7 @@ def test_chunked_local_attention_possible_cached_prefix():
kv_cache_spec=chunked_local_attention_spec,
drop_eagle_block=False,
alignment_tokens=block_size,
)[0]
)[0][0]
assert len(computed_blocks) == expect_length
assert all(
@@ -164,7 +164,7 @@ def test_sliding_window_possible_cached_prefix():
kv_cache_spec=sliding_window_spec,
drop_eagle_block=False,
alignment_tokens=block_size,
)[0]
)[0][0]
assert len(computed_blocks) == expect_length
assert all(
@@ -398,13 +398,13 @@ def test_get_num_blocks_to_allocate():
assert (
manager.get_num_blocks_to_allocate(
"1", 20 * block_size, cached_blocks_1, 0, 20 * block_size
"1", 20 * block_size, cached_blocks_1, 0, 0, 20 * block_size
)
== 20
)
assert (
manager.get_num_blocks_to_allocate(
"2", 20 * block_size, cached_blocks_2, 0, 20 * block_size
"2", 20 * block_size, cached_blocks_2, 0, 0, 20 * block_size
)
== 15
)
@@ -434,6 +434,7 @@ def test_evictable_cached_blocks_not_double_allocated():
num_tokens=2 * block_size,
new_computed_blocks=[evictable_block],
total_computed_tokens=block_size,
num_local_computed_tokens=block_size,
num_tokens_main_model=2 * block_size,
)
# Free capacity check should count evictable cached blocks, but allocation
@@ -474,13 +475,13 @@ def test_chunked_local_attention_get_num_blocks_to_allocate():
assert (
manager.get_num_blocks_to_allocate(
"1", 20 * block_size, cached_blocks_1, 0, 20 * block_size
"1", 20 * block_size, cached_blocks_1, 0, 0, 20 * block_size
)
== 20
)
assert (
manager.get_num_blocks_to_allocate(
"2", 20 * block_size, cached_blocks_2, 0, 20 * block_size
"2", 20 * block_size, cached_blocks_2, 0, 0, 20 * block_size
)
== 15
)
@@ -524,6 +525,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
num_tokens=num_tokens,
new_computed_blocks=[],
total_computed_tokens=total_computed,
num_local_computed_tokens=0,
num_tokens_main_model=num_tokens,
)
new_blocks = manager.allocate_new_blocks(
@@ -37,7 +37,7 @@ def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=Non
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
cmap = ExternalCachedBlockPool()
cmap = ExternalCachedBlockPool(16)
h = BlockHash(b"\xaa" * 4)
res = cmap.get_cached_block(h, [0, 1])
assert res is not None
@@ -48,7 +48,7 @@ def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
def test_external_cached_block_pool_hit_all_groups():
h = BlockHash(b"\x11\x22\x33\x44")
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)), (1, bytes(h))})
res = cmap.get_cached_block(h, [0, 1])
assert res is not None
assert len(res) == 2
@@ -58,14 +58,14 @@ def test_external_cached_block_pool_hit_all_groups():
def test_external_cached_block_pool_miss_one_group():
h = BlockHash(b"\x11\x22\x33\x44")
cmap = ExternalCachedBlockPool({(0, bytes(h))})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h))})
assert cmap.get_cached_block(h, [0, 1]) is None
def test_external_cached_block_pool_unknown_hash():
h_known = BlockHash(b"\x01" * 4)
h_unknown = BlockHash(b"\x02" * 4)
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h_known))})
assert cmap.get_cached_block(h_unknown, [0]) is None
@@ -103,7 +103,7 @@ def test_coordinator_single_full_attention_all_hits():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 64
assert masks[0] == [True, True, True, True]
@@ -113,7 +113,7 @@ def test_coordinator_single_full_attention_partial_prefix():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
cmap = ExternalCachedBlockPool(16, {(0, bytes(hs[0])), (0, bytes(hs[1]))})
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 32
assert masks[0] == [True, True]
@@ -123,7 +123,7 @@ def test_coordinator_single_full_attention_no_hits():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool(set())
cmap = ExternalCachedBlockPool(16, set())
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 0
assert masks[0] == []
@@ -135,7 +135,7 @@ def test_coordinator_single_swa_tautological_pool_masks_pre_window():
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4) # 4 chunks * 16 tokens
cmap = ExternalCachedBlockPool()
cmap = ExternalCachedBlockPool(16)
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
assert hit == 64
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
@@ -153,7 +153,7 @@ def test_coordinator_hybrid_full_plus_swa_all_hit():
]
coord = _make_coord(groups, hash_block_size=16)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
cmap = ExternalCachedBlockPool(16, {(g, bytes(h)) for g in (0, 1) for h in hs})
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -169,7 +169,7 @@ def test_coordinator_hybrid_hole_in_full_clips_both():
hs = _hashes(4)
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
exists |= {(1, bytes(h)) for h in hs}
cmap = ExternalCachedBlockPool(exists)
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -188,7 +188,7 @@ def test_coordinator_group_block_size_double_hash():
big_hashes = list(chunk_hashes_for_block_size(hs, 16, 32))
exists = {(0, bytes(h)) for h in hs}
exists |= {(1, bytes(bh)) for bh in big_hashes}
cmap = ExternalCachedBlockPool(exists)
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -405,7 +405,7 @@ def test_lookup_with_eagle_pops_last_full_attention_block():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -426,7 +426,7 @@ def test_load_mask_with_eagle_does_not_double_prune_full_attention():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -450,7 +450,7 @@ def test_load_mask_with_eagle_hybrid_full_plus_swa():
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
hs = _hashes(4)
exists = {(g, bytes(h)) for g in (0, 1) for h in hs}
cmap = ExternalCachedBlockPool(exists)
cmap = ExternalCachedBlockPool(16, exists)
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -469,7 +469,7 @@ def test_load_mask_without_eagle_unchanged():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16, use_eagle=False)
hs = _hashes(4)
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
_masks, hit = coord.find_longest_cache_hit(
hs, max_length=64, cached_block_pool=cmap
)
@@ -65,6 +65,7 @@ def _minimal_vllm_config(cache_block_size=16):
cfg.cache_config.block_size = cache_block_size
cfg.cache_config.num_gpu_blocks = 4
cfg.cache_config.hash_block_size = None
cfg.cache_config.prefix_match_unit = None
cfg.cache_config.enable_prefix_caching = True
cfg.parallel_config.prefill_context_parallel_size = 1
cfg.parallel_config.decode_context_parallel_size = 1