[2/N][Core] support partial prefix cache hit for hybrid model (#46384)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
8e981630c9
commit
481e481be7
@@ -0,0 +1,816 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fine-grained partial prefix-cache hits for hybrid (full attention + mamba
|
||||
"align") models: scheduler chunk splitting, partial tail registration, CoW
|
||||
on partial hits, and same-step deferral."""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.v1.core.test_prefix_caching import make_kv_cache_manager, make_request
|
||||
from vllm.utils.hashing import sha256
|
||||
from vllm.v1.core.kv_cache_utils import (
|
||||
KVCacheBlockCopy,
|
||||
get_block_hash,
|
||||
get_group_id,
|
||||
init_none_hash,
|
||||
)
|
||||
from vllm.v1.core.sched.scheduler import Scheduler
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheConfig,
|
||||
KVCacheGroupSpec,
|
||||
MambaSpec,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _auto_init_hash_fn():
|
||||
init_none_hash(sha256)
|
||||
|
||||
|
||||
def test_mamba_align_split_partial_tail_schedule():
|
||||
"""Chunk ends with partial hits on: block-aligned chunks, one extra stop
|
||||
at the prompt's last hash boundary (registering the partial tail), then
|
||||
the remaining tokens. block=512, hash=32, prompt=10000, budget=8192:
|
||||
0 -> 8192 -> 9728 -> 9984 -> 10000."""
|
||||
block_size = 512
|
||||
hash_block_size = 32
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
use_eagle=False,
|
||||
hash_block_size=hash_block_size,
|
||||
mamba_partial_cache_hit=True,
|
||||
)
|
||||
split = Scheduler._mamba_block_aligned_split
|
||||
|
||||
req = make_request("0", [0] * 10000, hash_block_size, sha256)
|
||||
req.num_computed_tokens = 0
|
||||
assert split(self=mock, request=req, num_new_tokens=8192) == 8192
|
||||
req.num_computed_tokens = 8192
|
||||
# Stop at the last block boundary (9728).
|
||||
assert split(self=mock, request=req, num_new_tokens=1808) == 1536
|
||||
req.num_computed_tokens = 9728
|
||||
# Extra stop at the prompt's last hash boundary (9984).
|
||||
assert split(self=mock, request=req, num_new_tokens=272) == 256
|
||||
req.num_computed_tokens = 9984
|
||||
# Final 16 tokens run unchanged (no mid-block-resume stop: the next
|
||||
# block boundary is past the last block boundary).
|
||||
assert split(self=mock, request=req, num_new_tokens=16) == 16
|
||||
|
||||
# Partial hits off: no extra stop, the tail runs in one chunk.
|
||||
mock.mamba_partial_cache_hit = False
|
||||
req.num_computed_tokens = 9728
|
||||
assert split(self=mock, request=req, num_new_tokens=272) == 272
|
||||
mock.mamba_partial_cache_hit = True
|
||||
|
||||
# A request resumed mid-block (partial hash hit at 9984): the first chunk
|
||||
# stops at the next block boundary (10240), later chunk ends re-align.
|
||||
req2 = make_request("1", [0] * 12000, hash_block_size, sha256)
|
||||
req2.num_computed_tokens = 9984
|
||||
assert split(self=mock, request=req2, num_new_tokens=2016) == 256
|
||||
req2.num_computed_tokens = 10240
|
||||
assert split(self=mock, request=req2, num_new_tokens=1000) == 512
|
||||
|
||||
|
||||
def test_hybrid_mamba_align_partial_hash_hit():
|
||||
hash_block_size = 2
|
||||
mamba_block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=20,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=mamba_block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
blocks = manager.allocate_slots(req0, 6, num_computed, computed_blocks)
|
||||
assert blocks is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert partial_mamba_block is not None
|
||||
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in computed_blocks.blocks] == [3, 2]
|
||||
|
||||
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||
assert new_blocks is not None
|
||||
mamba_new_block_ids = new_blocks.get_block_ids()[1]
|
||||
assert len(mamba_new_block_ids) == 1
|
||||
assert mamba_new_block_ids[0] != partial_mamba_block[0].block_id
|
||||
assert manager.get_blocks("1").get_block_ids()[1][1] == mamba_new_block_ids[0]
|
||||
assert partial_mamba_block[0].block_hash is not None
|
||||
assert get_block_hash(partial_mamba_block[0].block_hash) == partial_mamba_hash
|
||||
assert get_group_id(partial_mamba_block[0].block_hash) == 1
|
||||
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||
copies, _ = manager.take_kv_cache_block_copies()
|
||||
assert (
|
||||
KVCacheBlockCopy(
|
||||
src_block_id=partial_mamba_block[0].block_id,
|
||||
dst_block_id=mamba_new_block_ids[0],
|
||||
)
|
||||
in copies
|
||||
)
|
||||
assert manager.get_blocks("1").blocks[1][1].block_hash_num_tokens == 8
|
||||
|
||||
|
||||
def test_hybrid_mamba_partial_tail_owner_uses_cow_on_continue():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert partial_mamba_block is not None
|
||||
partial_mamba_block_id = partial_mamba_block[0].block_id
|
||||
assert manager.get_blocks("0").get_block_ids()[1][1] == partial_mamba_block_id
|
||||
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
new_blocks = manager.allocate_slots(req0, 1)
|
||||
assert new_blocks is not None
|
||||
|
||||
# Reversed CoW for the owning request: it keeps its own block (the
|
||||
# worker's block table is append-only), and no new mamba block is handed
|
||||
# to the worker. The prefix-cache entry is moved to a private copy that
|
||||
# the queued block copy fills before the next forward.
|
||||
assert new_blocks.get_block_ids()[1] == []
|
||||
assert manager.get_blocks("0").get_block_ids()[1][1] == partial_mamba_block_id
|
||||
copies, _ = manager.take_kv_cache_block_copies()
|
||||
cow_copy = next(c for c in copies if c.src_block_id == partial_mamba_block_id)
|
||||
assert cow_copy.dst_block_id != partial_mamba_block_id
|
||||
# The source block gave up the hash; the copy target now owns the entry.
|
||||
assert partial_mamba_block[0].block_hash is None
|
||||
moved = manager.block_pool.get_cached_block(
|
||||
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert moved is not None
|
||||
assert moved[0].block_id == cow_copy.dst_block_id
|
||||
assert get_block_hash(moved[0].block_hash) == partial_mamba_hash
|
||||
assert get_group_id(moved[0].block_hash) == 1
|
||||
assert moved[0].block_hash_num_tokens == 6
|
||||
|
||||
|
||||
def test_hybrid_mamba_partial_tail_owner_continue_preserves_later_hit():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
partial_mamba_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||
partial_mamba_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert partial_mamba_block is not None
|
||||
partial_mamba_block_id = partial_mamba_block[0].block_id
|
||||
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
# The owner moved the prefix-cache entry to a private copy; capture its id.
|
||||
owner_copies, _ = manager.take_kv_cache_block_copies()
|
||||
cow_copy = next(c for c in owner_copies if c.src_block_id == partial_mamba_block_id)
|
||||
moved_block_id = cow_copy.dst_block_id
|
||||
manager.new_step_starts()
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
# The later request hits the moved (private-copy) entry, not the source.
|
||||
assert computed_blocks.get_block_ids()[1][1] == moved_block_id
|
||||
|
||||
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||
assert new_blocks is not None
|
||||
mamba_new_block_ids = new_blocks.get_block_ids()[1]
|
||||
assert len(mamba_new_block_ids) == 1
|
||||
assert mamba_new_block_ids[0] != moved_block_id
|
||||
# The hitting request CoWs from the moved entry into its own private block.
|
||||
copies, _ = manager.take_kv_cache_block_copies()
|
||||
assert (
|
||||
KVCacheBlockCopy(
|
||||
src_block_id=moved_block_id,
|
||||
dst_block_id=mamba_new_block_ids[0],
|
||||
)
|
||||
in copies
|
||||
)
|
||||
|
||||
|
||||
def test_hybrid_mamba_moved_partial_entry_defers_same_step_hit():
|
||||
"""The owner's move re-arms the same-step guard: the moved entry is
|
||||
filled by this step's copy, and chained same-step copies read stale
|
||||
sources, so a request hitting it in the move step must be deferred."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.new_step_starts()
|
||||
|
||||
# The owning request continues decoding: the partial entry moves to a
|
||||
# private copy in this step.
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
|
||||
# A request hitting the moved entry in the SAME step must be deferred.
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 4, 4], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is None
|
||||
|
||||
# Next step the moved entry is consumable.
|
||||
manager.new_step_starts()
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert manager.allocate_slots(req1, 2, num_computed, computed_blocks) is not None
|
||||
|
||||
|
||||
def test_hybrid_full_attention_partial_hash_hit_uses_cow():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
partial_full_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
partial_full_block = manager.block_pool.get_cached_block(
|
||||
partial_full_hash, kv_cache_group_ids=[0]
|
||||
)
|
||||
assert partial_full_block is not None
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
|
||||
|
||||
new_blocks = manager.allocate_slots(req1, 2, num_computed, computed_blocks)
|
||||
assert new_blocks is not None
|
||||
full_new_block_ids = new_blocks.get_block_ids()[0]
|
||||
assert len(full_new_block_ids) == 1
|
||||
assert full_new_block_ids[0] != partial_full_block[0].block_id
|
||||
assert partial_full_block[0].block_hash is not None
|
||||
assert get_block_hash(partial_full_block[0].block_hash) == partial_full_hash
|
||||
assert get_group_id(partial_full_block[0].block_hash) == 0
|
||||
assert partial_full_block[0].block_hash_num_tokens == 6
|
||||
copies, retained = manager.take_kv_cache_block_copies()
|
||||
assert (
|
||||
KVCacheBlockCopy(
|
||||
src_block_id=partial_full_block[0].block_id,
|
||||
dst_block_id=full_new_block_ids[0],
|
||||
)
|
||||
in copies
|
||||
)
|
||||
assert partial_full_block[0].ref_cnt == 1
|
||||
manager.block_pool.free_blocks(retained)
|
||||
assert partial_full_block[0].ref_cnt == 0
|
||||
|
||||
|
||||
def test_hybrid_partial_hit_cow_target_starts_uncached():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert num_computed == 0
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
manager.free(req0)
|
||||
manager.new_step_starts()
|
||||
|
||||
partial_hash = req0.block_hashes[6 // hash_block_size - 1]
|
||||
partial_full_block = manager.block_pool.get_cached_block(
|
||||
partial_hash, kv_cache_group_ids=[0]
|
||||
)
|
||||
partial_mamba_block = manager.block_pool.get_cached_block(
|
||||
partial_hash, kv_cache_group_ids=[1]
|
||||
)
|
||||
assert partial_full_block is not None
|
||||
assert partial_mamba_block is not None
|
||||
|
||||
req1 = make_request("1", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 6
|
||||
|
||||
new_blocks = manager.allocate_slots(
|
||||
req1,
|
||||
2,
|
||||
num_computed,
|
||||
computed_blocks,
|
||||
delay_cache_blocks=True,
|
||||
)
|
||||
assert new_blocks is not None
|
||||
|
||||
full_cow_block = manager.get_blocks("1").blocks[0][1]
|
||||
mamba_cow_block = manager.get_blocks("1").blocks[1][1]
|
||||
assert full_cow_block.block_id != partial_full_block[0].block_id
|
||||
assert mamba_cow_block.block_id != partial_mamba_block[0].block_id
|
||||
assert full_cow_block.block_hash is None
|
||||
assert full_cow_block.block_hash_num_tokens is None
|
||||
assert mamba_cow_block.block_hash is None
|
||||
assert mamba_cow_block.block_hash_num_tokens is None
|
||||
|
||||
assert partial_full_block[0].block_hash is not None
|
||||
assert get_block_hash(partial_full_block[0].block_hash) == partial_hash
|
||||
assert get_group_id(partial_full_block[0].block_hash) == 0
|
||||
assert partial_full_block[0].block_hash_num_tokens == 6
|
||||
assert partial_mamba_block[0].block_hash is not None
|
||||
assert get_block_hash(partial_mamba_block[0].block_hash) == partial_hash
|
||||
assert get_group_id(partial_mamba_block[0].block_hash) == 1
|
||||
assert partial_mamba_block[0].block_hash_num_tokens == 6
|
||||
|
||||
|
||||
def test_hybrid_partial_hash_truncates_full_attention_hit_length():
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
pool = manager.block_pool
|
||||
req = make_request(
|
||||
"0",
|
||||
[0, 0, 1, 1, 2, 2, 3, 3, 4, 4, 5, 5],
|
||||
hash_block_size,
|
||||
sha256,
|
||||
)
|
||||
|
||||
full_blocks = pool.get_new_blocks(3)
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=full_blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=2,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
pool.cache_partial_block(
|
||||
request=req,
|
||||
block=full_blocks[2],
|
||||
num_tokens=10,
|
||||
kv_cache_group_id=0,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
mamba_block = pool.get_new_blocks(1)[0]
|
||||
pool.cache_partial_block(
|
||||
request=req,
|
||||
block=mamba_block,
|
||||
num_tokens=6,
|
||||
kv_cache_group_id=1,
|
||||
block_size=block_size,
|
||||
)
|
||||
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req)
|
||||
assert num_computed == 6
|
||||
assert [len(group) for group in computed_blocks.blocks] == [2, 2]
|
||||
|
||||
|
||||
def test_cow_retained_blocks_returned_for_release():
|
||||
"""new_step_starts returns the CoW copy retentions instead of freeing
|
||||
them; the scheduler owns releasing them once the copy has run."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=24,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=hash_block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
)
|
||||
req0 = make_request("0", [0, 0, 1, 1, 2, 2], hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 6, num_computed, computed_blocks) is not None
|
||||
|
||||
# The owner's move queues a copy and retains both endpoints.
|
||||
req0.num_computed_tokens = 6
|
||||
req0.append_output_token_ids([3])
|
||||
assert manager.allocate_slots(req0, 1) is not None
|
||||
(cow_copy,), retained = manager.take_kv_cache_block_copies()
|
||||
assert {b.block_id for b in retained} == {
|
||||
cow_copy.src_block_id,
|
||||
cow_copy.dst_block_id,
|
||||
}
|
||||
# Not freed yet: the retention refs are still held.
|
||||
assert all(b.ref_cnt > 0 for b in retained)
|
||||
manager.block_pool.free_blocks(retained)
|
||||
|
||||
|
||||
def test_free_cow_retained_blocks_defers_until_copy_step_processed():
|
||||
"""Scheduler releases CoW retentions immediately when the copy's step has
|
||||
been processed (or deferral is off), and defers them otherwise."""
|
||||
from collections import deque
|
||||
|
||||
freed: list = []
|
||||
blocks = [SimpleNamespace(block_id=7), SimpleNamespace(block_id=9)]
|
||||
mock = SimpleNamespace(
|
||||
kv_cache_manager=SimpleNamespace(
|
||||
block_pool=SimpleNamespace(free_blocks=freed.extend)
|
||||
),
|
||||
deferred_frees=deque(),
|
||||
defer_block_free=True,
|
||||
processed_step_seq=2,
|
||||
)
|
||||
free = Scheduler._free_cow_retained_blocks
|
||||
|
||||
# Copy step still in flight: deferred with its fence.
|
||||
free(mock, list(blocks), fence_seq=3)
|
||||
assert not freed
|
||||
assert mock.deferred_frees == deque([(3, blocks[::-1])])
|
||||
|
||||
# Copy step processed: freed immediately.
|
||||
mock.processed_step_seq = 3
|
||||
free(mock, list(blocks), fence_seq=3)
|
||||
assert freed == blocks
|
||||
|
||||
# Deferral disabled: freed immediately regardless of the fence.
|
||||
freed.clear()
|
||||
mock.deferred_frees.clear()
|
||||
mock.defer_block_free = False
|
||||
mock.processed_step_seq = 0
|
||||
free(mock, list(blocks), fence_seq=3)
|
||||
assert freed == blocks
|
||||
|
||||
|
||||
def test_full_attention_eagle_drops_one_hash_unit():
|
||||
"""With fine-grained partial hits, eagle rewinds the hit by one hash unit
|
||||
instead of a whole cache block: the tail block's KV is append-only, so it
|
||||
still covers the reduced length and stays in the hit as a partial block."""
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.single_type_kv_cache_manager import FullAttentionManager
|
||||
|
||||
hash_block_size = 2
|
||||
block_size = 4
|
||||
pool = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=hash_block_size
|
||||
)
|
||||
spec = FullAttentionSpec(
|
||||
block_size=block_size, num_kv_heads=1, head_size=1, dtype=torch.float32
|
||||
)
|
||||
req = make_request("0", [0, 0, 1, 1, 2, 2, 3, 3], hash_block_size, sha256)
|
||||
|
||||
def find(drop_eagle_block):
|
||||
return FullAttentionManager.find_longest_cache_hit(
|
||||
block_hashes=req.block_hashes,
|
||||
max_length=8,
|
||||
kv_cache_group_ids=[0],
|
||||
block_pool=pool,
|
||||
kv_cache_spec=spec,
|
||||
drop_eagle_block=drop_eagle_block,
|
||||
alignment_tokens=hash_block_size,
|
||||
)
|
||||
|
||||
# Two full cached blocks (hit 8): eagle rewinds to 6, keeping the last
|
||||
# block as a partial hit instead of dropping it to 4.
|
||||
blocks = pool.get_new_blocks(2)
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks,
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=2,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
hit_blocks, hit_length = find(drop_eagle_block=False)
|
||||
assert (hit_length, len(hit_blocks[0])) == (8, 2)
|
||||
hit_blocks, hit_length = find(drop_eagle_block=True)
|
||||
assert (hit_length, len(hit_blocks[0])) == (6, 2)
|
||||
|
||||
# A partial tail at 6 (block 1 not fully cached): eagle rewinds to the
|
||||
# block boundary and trims the tail block.
|
||||
pool2 = BlockPool(
|
||||
num_gpu_blocks=10, enable_caching=True, hash_block_size=hash_block_size
|
||||
)
|
||||
pool = pool2
|
||||
blocks = pool.get_new_blocks(2)
|
||||
pool.cache_full_blocks(
|
||||
request=req,
|
||||
blocks=blocks[:1],
|
||||
num_cached_blocks=0,
|
||||
num_full_blocks=1,
|
||||
block_size=block_size,
|
||||
kv_cache_group_id=0,
|
||||
)
|
||||
assert (
|
||||
pool.cache_partial_block(
|
||||
request=req,
|
||||
block=blocks[1],
|
||||
num_tokens=6,
|
||||
kv_cache_group_id=0,
|
||||
block_size=block_size,
|
||||
)
|
||||
is not None
|
||||
)
|
||||
hit_blocks, hit_length = find(drop_eagle_block=False)
|
||||
assert (hit_length, len(hit_blocks[0])) == (6, 2)
|
||||
hit_blocks, hit_length = find(drop_eagle_block=True)
|
||||
assert (hit_length, len(hit_blocks[0])) == (4, 1)
|
||||
|
||||
|
||||
def test_hybrid_partial_hit_with_eagle_stays_within_group_blocks():
|
||||
"""Regression: with eagle, the mamba group must not receive the eagle
|
||||
lookup margin — its finder never applies the drop, so it could return a
|
||||
hit past the blocks the (dropped) full-attention group covers, crashing
|
||||
the consumer's CoW with block_idx >= len(req_blocks)."""
|
||||
hash_block_size = 2
|
||||
block_size = 2 * hash_block_size
|
||||
kv_cache_config = KVCacheConfig(
|
||||
num_blocks=32,
|
||||
kv_cache_tensors=[],
|
||||
kv_cache_groups=[
|
||||
KVCacheGroupSpec(
|
||||
["full"],
|
||||
FullAttentionSpec(
|
||||
block_size=block_size,
|
||||
num_kv_heads=1,
|
||||
head_size=1,
|
||||
dtype=torch.float32,
|
||||
),
|
||||
),
|
||||
KVCacheGroupSpec(
|
||||
["mamba"],
|
||||
MambaSpec(
|
||||
block_size=block_size,
|
||||
shapes=(1, 1),
|
||||
dtypes=(torch.float32,),
|
||||
mamba_cache_mode="align",
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
manager = make_kv_cache_manager(
|
||||
kv_cache_config=kv_cache_config,
|
||||
max_model_len=8192,
|
||||
enable_caching=True,
|
||||
hash_block_size=hash_block_size,
|
||||
use_eagle=True,
|
||||
)
|
||||
|
||||
# The owner prefills in scheduler-split style: stop at the block boundary
|
||||
# (4), then at the prompt's last hash boundary (6, partial entries).
|
||||
req0 = make_request("0", [7] * 6, hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req0)
|
||||
assert manager.allocate_slots(req0, 4, num_computed, computed_blocks) is not None
|
||||
req0.num_computed_tokens = 4
|
||||
manager.new_step_starts()
|
||||
assert manager.allocate_slots(req0, 2) is not None
|
||||
req0.num_computed_tokens = 6
|
||||
manager.new_step_starts()
|
||||
|
||||
# A longer request with eagle: full attention drops the partial tail, so
|
||||
# the joint hit must fall back to the block boundary the FA blocks cover.
|
||||
req1 = make_request("1", [7] * 6 + [9] * 2, hash_block_size, sha256)
|
||||
computed_blocks, num_computed = manager.get_computed_blocks(req1)
|
||||
assert num_computed == 4
|
||||
assert all(
|
||||
len(group) * block_size >= num_computed for group in computed_blocks.blocks
|
||||
)
|
||||
assert manager.allocate_slots(req1, 4, num_computed, computed_blocks) is not None
|
||||
@@ -18,6 +18,7 @@ from unittest.mock import PropertyMock, patch
|
||||
import pytest
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.v1.core.kv_cache_utils import KVCacheBlockCopy
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.outputs import ModelRunnerOutput
|
||||
from vllm.v1.request import RequestStatus
|
||||
@@ -412,3 +413,64 @@ def test_non_async_abort_defers_via_last_sched_seq():
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
assert not scheduler.deferred_frees
|
||||
assert pool.get_num_free_blocks() == num_free_initially
|
||||
|
||||
|
||||
def test_cow_retentions_deferred_until_copy_step_processed():
|
||||
"""The endpoints of a queued KV block copy must stay out of the free
|
||||
pool until the step that runs the copy has been processed. Freed
|
||||
earlier, an endpoint can be reallocated (e.g. as a PD KV-load
|
||||
destination) and overwritten by a transfer that is not ordered against
|
||||
the copy still pending in the in-flight step.
|
||||
"""
|
||||
scheduler = _create_deferring_scheduler()
|
||||
pool = scheduler.kv_cache_manager.block_pool
|
||||
manager = scheduler.kv_cache_manager.coordinator.single_type_managers[0]
|
||||
|
||||
request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=NUM_PROMPT_TOKENS,
|
||||
max_tokens=5,
|
||||
stop_token_ids=[STOP_TOKEN_ID],
|
||||
)[0]
|
||||
scheduler.add_request(request)
|
||||
|
||||
# Simulate a partial-hit CoW performed while scheduling step 1, whose
|
||||
# hitting request was freed within the same step: the copy rides out0
|
||||
# and each endpoint stays alive only through its copy retention.
|
||||
src_block, dst_block = pool.get_new_blocks(2)
|
||||
block_copy = KVCacheBlockCopy(
|
||||
src_block_id=src_block.block_id, dst_block_id=dst_block.block_id
|
||||
)
|
||||
manager._pending_cow_copies.append((src_block, dst_block))
|
||||
out0 = scheduler.schedule()
|
||||
assert out0.kv_cache_block_copies == [block_copy]
|
||||
|
||||
# Exhaust the rest of the pool so the copy endpoints are the only blocks
|
||||
# a new request could receive, then add one that fits exactly in them.
|
||||
pool.get_new_blocks(pool.get_num_free_blocks())
|
||||
late_request = create_requests(
|
||||
num_requests=1,
|
||||
num_tokens=2 * scheduler.block_size,
|
||||
max_tokens=5,
|
||||
req_ids=["late"],
|
||||
)[0]
|
||||
scheduler.add_request(late_request)
|
||||
|
||||
# Step 2 is scheduled while step 1 (which runs the copy) is still in
|
||||
# flight: the retentions are released against the copy's fence, so the
|
||||
# endpoints must not reach the free pool -- the late request must not be
|
||||
# scheduled onto them.
|
||||
out1 = scheduler.schedule()
|
||||
assert src_block.ref_cnt == 1
|
||||
assert dst_block.ref_cnt == 1
|
||||
assert scheduler.deferred_frees
|
||||
assert not out1.scheduled_new_reqs
|
||||
|
||||
# Step 1's output is processed: the copy has run, endpoints return to
|
||||
# the pool and the late request can be scheduled onto them safely.
|
||||
scheduler.update_from_output(out0, _make_model_runner_output(out0))
|
||||
assert src_block.ref_cnt == 0
|
||||
assert dst_block.ref_cnt == 0
|
||||
assert not scheduler.deferred_frees
|
||||
out2 = scheduler.schedule()
|
||||
assert [r.req_id for r in out2.scheduled_new_reqs] == ["late"]
|
||||
|
||||
@@ -2597,3 +2597,63 @@ def test_hma_not_disabled_when_kv_events_enabled():
|
||||
assert vllm_config.scheduler_config.disable_hybrid_kv_cache_manager is False, (
|
||||
"kv_events_config must not force-disable the hybrid KV cache manager."
|
||||
)
|
||||
|
||||
|
||||
def test_resolve_block_hashes_gate():
|
||||
# Resolve symbols through the module so they stay consistent with each other
|
||||
# even after other tests reload ``kv_cache_utils``.
|
||||
resolve_block_hashes = kv_cache_utils.resolve_block_hashes
|
||||
BlockHashListWithBlockSize = kv_cache_utils.BlockHashListWithBlockSize
|
||||
# Raw, hash_block_size-granularity hashes (contents are opaque here).
|
||||
raw = [BlockHash(bytes([i])) for i in range(8)]
|
||||
|
||||
# block_size == hash_block_size: always reuse the raw hashes.
|
||||
assert resolve_block_hashes(raw, 2, 2, alignment_tokens=2) is raw
|
||||
assert (
|
||||
resolve_block_hashes(
|
||||
raw, 2, 2, supports_fine_grained_hash_lookup=True, alignment_tokens=2
|
||||
)
|
||||
is raw
|
||||
)
|
||||
|
||||
# Fine-grained manager, partial hits ON (alignment_tokens == hash_block_size
|
||||
# < block_size): keep raw hashes so the manager can scan at hash granularity.
|
||||
assert (
|
||||
resolve_block_hashes(
|
||||
raw, 2, 4, supports_fine_grained_hash_lookup=True, alignment_tokens=2
|
||||
)
|
||||
is raw
|
||||
)
|
||||
|
||||
# Fine-grained manager, partial hits OFF (alignment_tokens ==
|
||||
# scheduler_block_size >= block_size): must fall back to a block-size view,
|
||||
# exactly like the pre-refactor coordinator did.
|
||||
off = resolve_block_hashes(
|
||||
raw, 2, 4, supports_fine_grained_hash_lookup=True, alignment_tokens=4
|
||||
)
|
||||
assert isinstance(off, BlockHashListWithBlockSize)
|
||||
assert off.scale_factor == 2
|
||||
|
||||
# Non-fine-grained manager (e.g. sliding window): always a block-size view
|
||||
# when block_size != hash_block_size, regardless of alignment_tokens.
|
||||
swa = resolve_block_hashes(
|
||||
raw, 2, 4, supports_fine_grained_hash_lookup=False, alignment_tokens=2
|
||||
)
|
||||
assert isinstance(swa, BlockHashListWithBlockSize)
|
||||
assert swa.scale_factor == 2
|
||||
|
||||
|
||||
def test_resolve_block_hashes_rejects_mismatched_view():
|
||||
resolve_block_hashes = kv_cache_utils.resolve_block_hashes
|
||||
BlockHashListWithBlockSize = kv_cache_utils.BlockHashListWithBlockSize
|
||||
raw = [BlockHash(bytes([i])) for i in range(8)]
|
||||
|
||||
# A view built at this block_size is returned as-is (idempotent).
|
||||
view = BlockHashListWithBlockSize(raw, 2, 4)
|
||||
assert resolve_block_hashes(view, 2, 4) is view
|
||||
|
||||
# A view built at a different block_size must fail loudly rather than be
|
||||
# silently reinterpreted at the wrong granularity.
|
||||
mismatched = BlockHashListWithBlockSize(raw, 2, 8)
|
||||
with pytest.raises(AssertionError):
|
||||
resolve_block_hashes(mismatched, 2, 4)
|
||||
|
||||
@@ -1072,7 +1072,10 @@ def test_hybrid_cache_mamba_align_shared_prefix_detection():
|
||||
# Next, validate scheduler logic for num_uncached_common_prefix_tokens > 0
|
||||
# Create minimal mock with just the needed attributes
|
||||
mock = SimpleNamespace(
|
||||
cache_config=SimpleNamespace(block_size=block_size), use_eagle=False
|
||||
cache_config=SimpleNamespace(block_size=block_size),
|
||||
use_eagle=False,
|
||||
hash_block_size=block_size,
|
||||
mamba_partial_cache_hit=False,
|
||||
)
|
||||
num_new_tokens_adjusted = Scheduler._mamba_block_aligned_split(
|
||||
self=mock,
|
||||
|
||||
@@ -93,7 +93,7 @@ def test_chunked_local_attention_possible_cached_prefix():
|
||||
kv_cache_spec=chunked_local_attention_spec,
|
||||
drop_eagle_block=False,
|
||||
alignment_tokens=block_size,
|
||||
)[0]
|
||||
)[0][0]
|
||||
assert len(computed_blocks) == expect_length
|
||||
|
||||
assert all(
|
||||
@@ -164,7 +164,7 @@ def test_sliding_window_possible_cached_prefix():
|
||||
kv_cache_spec=sliding_window_spec,
|
||||
drop_eagle_block=False,
|
||||
alignment_tokens=block_size,
|
||||
)[0]
|
||||
)[0][0]
|
||||
assert len(computed_blocks) == expect_length
|
||||
|
||||
assert all(
|
||||
@@ -398,13 +398,13 @@ def test_get_num_blocks_to_allocate():
|
||||
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
"1", 20 * block_size, cached_blocks_1, 0, 20 * block_size
|
||||
"1", 20 * block_size, cached_blocks_1, 0, 0, 20 * block_size
|
||||
)
|
||||
== 20
|
||||
)
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
"2", 20 * block_size, cached_blocks_2, 0, 20 * block_size
|
||||
"2", 20 * block_size, cached_blocks_2, 0, 0, 20 * block_size
|
||||
)
|
||||
== 15
|
||||
)
|
||||
@@ -434,6 +434,7 @@ def test_evictable_cached_blocks_not_double_allocated():
|
||||
num_tokens=2 * block_size,
|
||||
new_computed_blocks=[evictable_block],
|
||||
total_computed_tokens=block_size,
|
||||
num_local_computed_tokens=block_size,
|
||||
num_tokens_main_model=2 * block_size,
|
||||
)
|
||||
# Free capacity check should count evictable cached blocks, but allocation
|
||||
@@ -474,13 +475,13 @@ def test_chunked_local_attention_get_num_blocks_to_allocate():
|
||||
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
"1", 20 * block_size, cached_blocks_1, 0, 20 * block_size
|
||||
"1", 20 * block_size, cached_blocks_1, 0, 0, 20 * block_size
|
||||
)
|
||||
== 20
|
||||
)
|
||||
assert (
|
||||
manager.get_num_blocks_to_allocate(
|
||||
"2", 20 * block_size, cached_blocks_2, 0, 20 * block_size
|
||||
"2", 20 * block_size, cached_blocks_2, 0, 0, 20 * block_size
|
||||
)
|
||||
== 15
|
||||
)
|
||||
@@ -524,6 +525,7 @@ def test_predictor_matches_allocator_blocks_calculation_with_admission_cap():
|
||||
num_tokens=num_tokens,
|
||||
new_computed_blocks=[],
|
||||
total_computed_tokens=total_computed,
|
||||
num_local_computed_tokens=0,
|
||||
num_tokens_main_model=num_tokens,
|
||||
)
|
||||
new_blocks = manager.allocate_new_blocks(
|
||||
|
||||
@@ -37,7 +37,7 @@ def _make_coord(groups, hash_block_size, use_eagle=False, retention_interval=Non
|
||||
|
||||
|
||||
def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
||||
cmap = ExternalCachedBlockPool()
|
||||
cmap = ExternalCachedBlockPool(16)
|
||||
h = BlockHash(b"\xaa" * 4)
|
||||
res = cmap.get_cached_block(h, [0, 1])
|
||||
assert res is not None
|
||||
@@ -48,7 +48,7 @@ def test_external_cached_block_pool_tautological_returns_present_for_any_hash():
|
||||
|
||||
def test_external_cached_block_pool_hit_all_groups():
|
||||
h = BlockHash(b"\x11\x22\x33\x44")
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)), (1, bytes(h))})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)), (1, bytes(h))})
|
||||
res = cmap.get_cached_block(h, [0, 1])
|
||||
assert res is not None
|
||||
assert len(res) == 2
|
||||
@@ -58,14 +58,14 @@ def test_external_cached_block_pool_hit_all_groups():
|
||||
|
||||
def test_external_cached_block_pool_miss_one_group():
|
||||
h = BlockHash(b"\x11\x22\x33\x44")
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h))})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h))})
|
||||
assert cmap.get_cached_block(h, [0, 1]) is None
|
||||
|
||||
|
||||
def test_external_cached_block_pool_unknown_hash():
|
||||
h_known = BlockHash(b"\x01" * 4)
|
||||
h_unknown = BlockHash(b"\x02" * 4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h_known))})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h_known))})
|
||||
assert cmap.get_cached_block(h_unknown, [0]) is None
|
||||
|
||||
|
||||
@@ -103,7 +103,7 @@ def test_coordinator_single_full_attention_all_hits():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 64
|
||||
assert masks[0] == [True, True, True, True]
|
||||
@@ -113,7 +113,7 @@ def test_coordinator_single_full_attention_partial_prefix():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(hs[0])), (0, bytes(hs[1]))})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(hs[0])), (0, bytes(hs[1]))})
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 32
|
||||
assert masks[0] == [True, True]
|
||||
@@ -123,7 +123,7 @@ def test_coordinator_single_full_attention_no_hits():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool(set())
|
||||
cmap = ExternalCachedBlockPool(16, set())
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 0
|
||||
assert masks[0] == []
|
||||
@@ -135,7 +135,7 @@ def test_coordinator_single_swa_tautological_pool_masks_pre_window():
|
||||
groups = [KVCacheGroupSpec(["L0"], _swa(block_size=16, sliding_window=32))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4) # 4 chunks * 16 tokens
|
||||
cmap = ExternalCachedBlockPool()
|
||||
cmap = ExternalCachedBlockPool(16)
|
||||
masks, hit = coord.find_longest_cache_hit(hs, max_length=64, cached_block_pool=cmap)
|
||||
assert hit == 64
|
||||
# ceil((sw-1)/block_size) = ceil(31/16) = 2 tail blocks.
|
||||
@@ -153,7 +153,7 @@ def test_coordinator_hybrid_full_plus_swa_all_hit():
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(g, bytes(h)) for g in (0, 1) for h in hs})
|
||||
cmap = ExternalCachedBlockPool(16, {(g, bytes(h)) for g in (0, 1) for h in hs})
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
@@ -169,7 +169,7 @@ def test_coordinator_hybrid_hole_in_full_clips_both():
|
||||
hs = _hashes(4)
|
||||
exists = {(0, bytes(hs[0])), (0, bytes(hs[2])), (0, bytes(hs[3]))}
|
||||
exists |= {(1, bytes(h)) for h in hs}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
cmap = ExternalCachedBlockPool(16, exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
@@ -188,7 +188,7 @@ def test_coordinator_group_block_size_double_hash():
|
||||
big_hashes = list(chunk_hashes_for_block_size(hs, 16, 32))
|
||||
exists = {(0, bytes(h)) for h in hs}
|
||||
exists |= {(1, bytes(bh)) for bh in big_hashes}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
cmap = ExternalCachedBlockPool(16, exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
@@ -405,7 +405,7 @@ def test_lookup_with_eagle_pops_last_full_attention_block():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
@@ -426,7 +426,7 @@ def test_load_mask_with_eagle_does_not_double_prune_full_attention():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
@@ -450,7 +450,7 @@ def test_load_mask_with_eagle_hybrid_full_plus_swa():
|
||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=True)
|
||||
hs = _hashes(4)
|
||||
exists = {(g, bytes(h)) for g in (0, 1) for h in hs}
|
||||
cmap = ExternalCachedBlockPool(exists)
|
||||
cmap = ExternalCachedBlockPool(16, exists)
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
@@ -469,7 +469,7 @@ def test_load_mask_without_eagle_unchanged():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16, use_eagle=False)
|
||||
hs = _hashes(4)
|
||||
cmap = ExternalCachedBlockPool({(0, bytes(h)) for h in hs})
|
||||
cmap = ExternalCachedBlockPool(16, {(0, bytes(h)) for h in hs})
|
||||
_masks, hit = coord.find_longest_cache_hit(
|
||||
hs, max_length=64, cached_block_pool=cmap
|
||||
)
|
||||
|
||||
@@ -65,6 +65,7 @@ def _minimal_vllm_config(cache_block_size=16):
|
||||
cfg.cache_config.block_size = cache_block_size
|
||||
cfg.cache_config.num_gpu_blocks = 4
|
||||
cfg.cache_config.hash_block_size = None
|
||||
cfg.cache_config.prefix_match_unit = None
|
||||
cfg.cache_config.enable_prefix_caching = True
|
||||
cfg.parallel_config.prefill_context_parallel_size = 1
|
||||
cfg.parallel_config.decode_context_parallel_size = 1
|
||||
|
||||
Reference in New Issue
Block a user