[2/N][Core] support partial prefix cache hit for hybrid model (#46384)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Yifan Qiao <yifanqiao@inferact.ai>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jiangyun Zhu
2026-07-12 05:37:51 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 8e981630c9
commit 481e481be7
21 changed files with 1653 additions and 177 deletions
@@ -29,10 +29,15 @@ from vllm.v1.kv_cache_spec_registry import KVCacheSpecRegistry
class ExternalCachedBlockPool:
"""Duck-typed BlockPool backed by a ``(group_id, hash)`` exists set."""
def __init__(self, exists: set[tuple[int, bytes]] | None = None) -> None:
def __init__(
self,
hash_block_size: int,
exists: set[tuple[int, bytes]] | None = None,
) -> None:
# ``exists=None`` is used on the recv side where hit_length is already
# determined and we just want each spec's manager to apply its own mask.
self._exists = exists
self.hash_block_size = hash_block_size
self.null_block = KVCacheBlock(block_id=0)
# Dummy ID 1 for present block for duck-typing.
self._present_block = KVCacheBlock(block_id=1)
@@ -166,7 +171,7 @@ class MooncakeStoreCoordinator:
masks, _ = self.find_longest_cache_hit(
block_hashes,
token_len,
ExternalCachedBlockPool(),
ExternalCachedBlockPool(self.hash_block_size),
apply_eagle=False,
)
return masks
@@ -270,7 +275,7 @@ class MooncakeStoreCoordinator:
if len(self.attention_groups) == 1:
spec, group_ids, manager_cls = self.attention_groups[0]
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks = manager_cls.find_longest_cache_hit(
hit_blocks, hit_length = manager_cls.find_longest_cache_hit(
block_hashes=hashes, # type: ignore[arg-type]
max_length=max_length,
kv_cache_group_ids=group_ids,
@@ -283,11 +288,12 @@ class MooncakeStoreCoordinator:
blocks_by_group: list[list[KVCacheBlock]] = [[] for _ in range(num_groups)]
for gid, blks in zip(group_ids, hit_blocks, strict=True):
blocks_by_group[gid] = blks
return tuple(blocks_by_group), len(hit_blocks[0]) * spec.block_size
return tuple(blocks_by_group), hit_length
num_groups = len(self.kv_cache_groups)
hit_length = max_length
hit_blocks_by_group: list[list[KVCacheBlock] | None] = [None] * num_groups
hit_length_by_group: list[int] = [0] * num_groups
is_simple_hybrid = len(self.attention_groups) == 2 and isinstance(
self.attention_groups[0][0], FullAttentionSpec
@@ -298,10 +304,11 @@ class MooncakeStoreCoordinator:
curr_hit_length = hit_length
for idx, (spec, group_ids, manager_cls) in enumerate(self.attention_groups):
cached = hit_blocks_by_group[group_ids[0]]
first_group_id = group_ids[0]
cached = hit_blocks_by_group[first_group_id]
if isinstance(spec, FullAttentionSpec) and cached is not None:
curr_hit_length = (
curr_hit_length // spec.block_size * spec.block_size
curr_hit_length = min(
curr_hit_length, hit_length_by_group[first_group_id]
)
continue
@@ -310,7 +317,7 @@ class MooncakeStoreCoordinator:
if drop_eagle_block:
_max_length = min(curr_hit_length + spec.block_size, max_length)
hashes = self.block_hashes_for_spec(block_hashes, spec)
hit_blocks = manager_cls.find_longest_cache_hit(
hit_blocks, _new_hit_length = manager_cls.find_longest_cache_hit(
block_hashes=hashes, # type: ignore[arg-type]
max_length=_max_length,
kv_cache_group_ids=group_ids,
@@ -319,7 +326,6 @@ class MooncakeStoreCoordinator:
drop_eagle_block=drop_eagle_block,
alignment_tokens=self.lcm_block_size,
)
_new_hit_length = len(hit_blocks[0]) * spec.block_size
if drop_eagle_block:
eagle_verified.add(idx)
elif _new_hit_length < curr_hit_length:
@@ -327,6 +333,7 @@ class MooncakeStoreCoordinator:
curr_hit_length = _new_hit_length
for gid, blocks in zip(group_ids, hit_blocks, strict=True):
hit_blocks_by_group[gid] = blocks
hit_length_by_group[gid] = _new_hit_length
if curr_hit_length >= hit_length:
break
@@ -343,6 +350,7 @@ class MooncakeStoreCoordinator:
full_blks = hit_blocks_by_group[gid]
assert full_blks is not None
del full_blks[num_blocks:]
hit_length_by_group[gid] = hit_length
return (
tuple(blks if blks is not None else [] for blks in hit_blocks_by_group),
@@ -1523,7 +1523,9 @@ class MooncakeStoreWorker:
}
_masks, hit_length = self.coord.find_longest_cache_hit(
block_hashes, token_len, ExternalCachedBlockPool(exists_set)
block_hashes,
token_len,
ExternalCachedBlockPool(self.hash_block_size, exists_set),
)
return hit_length