[Mooncake] Skip KV lookup for non-reachable SWA blocks (#45444)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
@@ -200,7 +200,7 @@ def test_store_mask_full_attention_all_true():
|
||||
groups = [KVCacheGroupSpec(["L0"], _full(16))]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks == ([True, True, True, True],)
|
||||
assert masks == (None,)
|
||||
|
||||
|
||||
def test_store_mask_zero_aligned_returns_empty_per_group():
|
||||
@@ -210,7 +210,7 @@ def test_store_mask_zero_aligned_returns_empty_per_group():
|
||||
]
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
masks = coord.store_mask(0)
|
||||
assert masks == ([], [])
|
||||
assert masks == (None, None)
|
||||
|
||||
|
||||
def test_store_mask_swa_only_window_around_each_lcm_boundary():
|
||||
@@ -224,7 +224,7 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
masks = coord.store_mask(64)
|
||||
# Full-attn: 2 chunks * 32 tokens.
|
||||
assert masks[0] == [True, True]
|
||||
assert masks[0] is None
|
||||
# SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored.
|
||||
assert masks[1] == [False, False, False, True, False, False, False, True]
|
||||
|
||||
@@ -237,7 +237,7 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
|
||||
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
|
||||
coord = _make_coord(groups, hash_block_size=8)
|
||||
masks = coord.store_mask(64)
|
||||
assert masks[0] == [True, True]
|
||||
assert masks[0] is None
|
||||
# Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3.
|
||||
# Boundary at 64: chunks 6 and 7. Others stay False.
|
||||
assert masks[1] == [False, False, True, True, False, False, True, True]
|
||||
@@ -265,12 +265,12 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
|
||||
masks = coord.store_mask(512)
|
||||
|
||||
# Full-MLA: 2 chunks of 256, both stored.
|
||||
assert masks[0] == [True, True]
|
||||
assert masks[0] is None
|
||||
# SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4.
|
||||
# Per-segment template = [F,F,T,T]; tiled twice.
|
||||
assert masks[1] == [False, False, True, True] * 2
|
||||
# SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True.
|
||||
assert masks[2] == [True] * 8
|
||||
assert masks[2] is None
|
||||
# SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64.
|
||||
# Last 4 of each 64-chunk segment True.
|
||||
assert masks[3] == ([False] * 60 + [True] * 4) * 2
|
||||
@@ -289,7 +289,7 @@ def test_store_mask_fast_path_all_block_sizes_equal_lcm():
|
||||
assert coord.lcm_block_size == 64
|
||||
masks = coord.store_mask(256)
|
||||
# Every block in every group is True — no sub-lcm filtering possible.
|
||||
assert masks == ([True] * 4, [True] * 4)
|
||||
assert masks == (None, None)
|
||||
|
||||
|
||||
def test_store_mask_fast_path_single_attention_group():
|
||||
@@ -300,7 +300,7 @@ def test_store_mask_fast_path_single_attention_group():
|
||||
coord = _make_coord(groups, hash_block_size=16)
|
||||
assert len(coord.attention_groups) == 1
|
||||
masks = coord.store_mask(64)
|
||||
assert masks == ([True] * 4, [True] * 4)
|
||||
assert masks == (None, None)
|
||||
|
||||
|
||||
# ----- store_mask with retention_interval (DSV4 sparse SWA checkpointing) -----
|
||||
@@ -319,7 +319,7 @@ def test_store_mask_dense_default_matches_every_lcm_boundary():
|
||||
boundary: tokens 32/64/96/128 -> chunks 3/7/11/15."""
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8)
|
||||
masks = coord.store_mask(128)
|
||||
assert masks[0] == [True, True, True, True]
|
||||
assert masks[0] is None
|
||||
assert masks[1] == [i % 4 == 3 for i in range(16)]
|
||||
|
||||
|
||||
@@ -329,7 +329,7 @@ def test_store_mask_retention_interval_sparsifies_swa_tails():
|
||||
boundaries at 32 and 96."""
|
||||
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
|
||||
masks = coord.store_mask(128)
|
||||
assert masks[0] == [True, True, True, True] # full attn unaffected
|
||||
assert masks[0] is None # full attn unaffected
|
||||
assert masks[1] == [i in (7, 15) for i in range(16)]
|
||||
|
||||
|
||||
|
||||
@@ -1233,6 +1233,67 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
|
||||
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
|
||||
|
||||
|
||||
def test_lookup_checks_all_potential_swa_hit_boundaries():
|
||||
"""Lookup should skip SWA chunks that can never validate a hit, but still
|
||||
check earlier aligned boundaries when sparse retention stores only the
|
||||
current request's replay boundary.
|
||||
"""
|
||||
from vllm.v1.kv_cache_interface import (
|
||||
FullAttentionSpec,
|
||||
KVCacheGroupSpec,
|
||||
SlidingWindowSpec,
|
||||
)
|
||||
|
||||
worker = _make_bare_worker(block_size=8)
|
||||
full = FullAttentionSpec(block_size=32, num_kv_heads=8, head_size=64, dtype=None)
|
||||
swa = SlidingWindowSpec(
|
||||
block_size=8, num_kv_heads=8, head_size=64, dtype=None, sliding_window=8
|
||||
)
|
||||
worker._kv_cache_groups = [
|
||||
KVCacheGroupSpec(["full"], full),
|
||||
KVCacheGroupSpec(["swa"], swa),
|
||||
]
|
||||
worker.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
|
||||
block_size=32,
|
||||
hash_block_size=8,
|
||||
),
|
||||
ChunkedTokenDatabase(
|
||||
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
|
||||
block_size=8,
|
||||
hash_block_size=8,
|
||||
),
|
||||
]
|
||||
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
|
||||
worker._kv_cache_groups,
|
||||
scheduler_block_size=32,
|
||||
hash_block_size=8,
|
||||
retention_interval=0,
|
||||
)
|
||||
# Candidate order: 3 full-attention chunks, then SWA chunks 3, 7, 11.
|
||||
# Only the first full chunk and the SWA chunk ending at token 32 exist, so
|
||||
# lookup should recover a 32-token external prefix hit. A sparse
|
||||
# prompt-specific store mask for num_prompt_tokens=96 would only check SWA
|
||||
# chunk 7 and miss this earlier reusable prefix.
|
||||
worker.store.batch_is_exist.return_value = [1, 0, 0, 1, 0, 0]
|
||||
|
||||
result = worker.lookup(
|
||||
96,
|
||||
[f"h{i}".encode() for i in range(12)],
|
||||
)
|
||||
|
||||
assert result == 32
|
||||
keys = worker.store.batch_is_exist.call_args.args[0]
|
||||
assert len(keys) == 6
|
||||
swa_keys = [key for key in keys if "@group:1@" in key]
|
||||
assert swa_keys == [
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6833",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6837",
|
||||
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@683131",
|
||||
]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# register_kv_caches tests
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@@ -172,19 +172,50 @@ class MooncakeStoreCoordinator:
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
num_prompt_tokens: int | None = None,
|
||||
) -> tuple[list[bool], ...]:
|
||||
"""Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of
|
||||
group ``g`` should be written to the store so a future cache hit can
|
||||
consume it.
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group store masks.
|
||||
|
||||
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
|
||||
written to the store so a future cache hit can consume it. ``None`` is
|
||||
the all-True sentinel.
|
||||
|
||||
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
|
||||
so the store retains exactly the blocks the local prefix cache would.
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
retention_interval=self.retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
|
||||
def lookup_mask(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
"""Per-group lookup masks.
|
||||
|
||||
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
|
||||
looked up as an aligned hit boundary. ``None`` is the all-True
|
||||
sentinel.
|
||||
"""
|
||||
return self._reachable_masks(
|
||||
aligned_token_len,
|
||||
retention_interval=None,
|
||||
num_prompt_tokens=None,
|
||||
)
|
||||
|
||||
def _reachable_masks(
|
||||
self,
|
||||
aligned_token_len: int,
|
||||
*,
|
||||
retention_interval: int | None,
|
||||
num_prompt_tokens: int | None,
|
||||
) -> tuple[list[bool] | None, ...]:
|
||||
assert aligned_token_len % self.lcm_block_size == 0, (
|
||||
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
|
||||
f"lcm_block_size ({self.lcm_block_size})"
|
||||
)
|
||||
masks: list[list[bool]] = []
|
||||
masks: list[list[bool] | None] = []
|
||||
for g_idx, g in enumerate(self.kv_cache_groups):
|
||||
spec = _unwrap_spec(g.kv_cache_spec)
|
||||
num_chunks = aligned_token_len // spec.block_size
|
||||
@@ -196,10 +227,12 @@ class MooncakeStoreCoordinator:
|
||||
alignment_tokens=self.lcm_block_size,
|
||||
kv_cache_spec=spec,
|
||||
use_eagle=g_idx in self.eagle_group_ids,
|
||||
retention_interval=self.retention_interval,
|
||||
retention_interval=retention_interval,
|
||||
num_prompt_tokens=num_prompt_tokens,
|
||||
)
|
||||
masks.append([True] * num_chunks if mask is None else mask)
|
||||
if mask is not None:
|
||||
assert len(mask) == num_chunks
|
||||
masks.append(mask)
|
||||
return tuple(masks)
|
||||
|
||||
def block_hashes_for_spec(
|
||||
|
||||
@@ -548,7 +548,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
for chunk_idx, (start, end, key) in enumerate(
|
||||
db.process_tokens(token_len, req_meta.block_hashes)
|
||||
):
|
||||
if chunk_idx >= len(mask) or not mask[chunk_idx]:
|
||||
if mask is not None and (
|
||||
chunk_idx >= len(mask) or not mask[chunk_idx]
|
||||
):
|
||||
continue
|
||||
starts.append(start)
|
||||
ends.append(end)
|
||||
@@ -1375,9 +1377,11 @@ class MooncakeStoreWorker:
|
||||
# candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i].
|
||||
candidate_keys: list[str] = []
|
||||
candidate_meta: list[tuple[int, bytes]] = []
|
||||
lookup_masks = self.coord.lookup_mask(token_len)
|
||||
tp_count = min(self.tp_size, self.num_kv_head)
|
||||
for g_idx, db in enumerate(self.token_dbs):
|
||||
spec_block_size = db.block_size
|
||||
lookup_mask = lookup_masks[g_idx]
|
||||
group_hashes = self.coord.block_hashes_for_spec(
|
||||
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
|
||||
)
|
||||
@@ -1385,6 +1389,10 @@ class MooncakeStoreWorker:
|
||||
start_idx = chunk_id * spec_block_size
|
||||
if start_idx >= token_len:
|
||||
break
|
||||
if lookup_mask is not None and (
|
||||
chunk_id >= len(lookup_mask) or not lookup_mask[chunk_id]
|
||||
):
|
||||
continue
|
||||
for tp in range(tp_count):
|
||||
for pp in range(self.pp_size):
|
||||
md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)
|
||||
|
||||
Reference in New Issue
Block a user