[Mooncake] Skip KV lookup for non-reachable SWA blocks (#45444)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-06-18 02:23:20 -07:00
committed by GitHub
parent 1e9f04da14
commit 5fd3b276f8
4 changed files with 120 additions and 18 deletions
@@ -200,7 +200,7 @@ def test_store_mask_full_attention_all_true():
groups = [KVCacheGroupSpec(["L0"], _full(16))]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(64)
assert masks == ([True, True, True, True],)
assert masks == (None,)
def test_store_mask_zero_aligned_returns_empty_per_group():
@@ -210,7 +210,7 @@ def test_store_mask_zero_aligned_returns_empty_per_group():
]
coord = _make_coord(groups, hash_block_size=16)
masks = coord.store_mask(0)
assert masks == ([], [])
assert masks == (None, None)
def test_store_mask_swa_only_window_around_each_lcm_boundary():
@@ -224,7 +224,7 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary():
coord = _make_coord(groups, hash_block_size=8)
masks = coord.store_mask(64)
# Full-attn: 2 chunks * 32 tokens.
assert masks[0] == [True, True]
assert masks[0] is None
# SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored.
assert masks[1] == [False, False, False, True, False, False, False, True]
@@ -237,7 +237,7 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm():
groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)]
coord = _make_coord(groups, hash_block_size=8)
masks = coord.store_mask(64)
assert masks[0] == [True, True]
assert masks[0] is None
# Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3.
# Boundary at 64: chunks 6 and 7. Others stay False.
assert masks[1] == [False, False, True, True, False, False, True, True]
@@ -265,12 +265,12 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa():
masks = coord.store_mask(512)
# Full-MLA: 2 chunks of 256, both stored.
assert masks[0] == [True, True]
assert masks[0] is None
# SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4.
# Per-segment template = [F,F,T,T]; tiled twice.
assert masks[1] == [False, False, True, True] * 2
# SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True.
assert masks[2] == [True] * 8
assert masks[2] is None
# SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64.
# Last 4 of each 64-chunk segment True.
assert masks[3] == ([False] * 60 + [True] * 4) * 2
@@ -289,7 +289,7 @@ def test_store_mask_fast_path_all_block_sizes_equal_lcm():
assert coord.lcm_block_size == 64
masks = coord.store_mask(256)
# Every block in every group is True — no sub-lcm filtering possible.
assert masks == ([True] * 4, [True] * 4)
assert masks == (None, None)
def test_store_mask_fast_path_single_attention_group():
@@ -300,7 +300,7 @@ def test_store_mask_fast_path_single_attention_group():
coord = _make_coord(groups, hash_block_size=16)
assert len(coord.attention_groups) == 1
masks = coord.store_mask(64)
assert masks == ([True] * 4, [True] * 4)
assert masks == (None, None)
# ----- store_mask with retention_interval (DSV4 sparse SWA checkpointing) -----
@@ -319,7 +319,7 @@ def test_store_mask_dense_default_matches_every_lcm_boundary():
boundary: tokens 32/64/96/128 -> chunks 3/7/11/15."""
coord = _make_coord(_retention_groups(), hash_block_size=8)
masks = coord.store_mask(128)
assert masks[0] == [True, True, True, True]
assert masks[0] is None
assert masks[1] == [i % 4 == 3 for i in range(16)]
@@ -329,7 +329,7 @@ def test_store_mask_retention_interval_sparsifies_swa_tails():
boundaries at 32 and 96."""
coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64)
masks = coord.store_mask(128)
assert masks[0] == [True, True, True, True] # full attn unaffected
assert masks[0] is None # full attn unaffected
assert masks[1] == [i in (7, 15) for i in range(16)]
@@ -1233,6 +1233,67 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present():
assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64
def test_lookup_checks_all_potential_swa_hit_boundaries():
"""Lookup should skip SWA chunks that can never validate a hit, but still
check earlier aligned boundaries when sparse retention stores only the
current request's replay boundary.
"""
from vllm.v1.kv_cache_interface import (
FullAttentionSpec,
KVCacheGroupSpec,
SlidingWindowSpec,
)
worker = _make_bare_worker(block_size=8)
full = FullAttentionSpec(block_size=32, num_kv_heads=8, head_size=64, dtype=None)
swa = SlidingWindowSpec(
block_size=8, num_kv_heads=8, head_size=64, dtype=None, sliding_window=8
)
worker._kv_cache_groups = [
KVCacheGroupSpec(["full"], full),
KVCacheGroupSpec(["swa"], swa),
]
worker.token_dbs = [
ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=0),
block_size=32,
hash_block_size=8,
),
ChunkedTokenDatabase(
KeyMetadata("test-model", 0, 0, 0, 0, group_id=1),
block_size=8,
hash_block_size=8,
),
]
worker.coord = mooncake_store_worker.MooncakeStoreCoordinator(
worker._kv_cache_groups,
scheduler_block_size=32,
hash_block_size=8,
retention_interval=0,
)
# Candidate order: 3 full-attention chunks, then SWA chunks 3, 7, 11.
# Only the first full chunk and the SWA chunk ending at token 32 exist, so
# lookup should recover a 32-token external prefix hit. A sparse
# prompt-specific store mask for num_prompt_tokens=96 would only check SWA
# chunk 7 and miss this earlier reusable prefix.
worker.store.batch_is_exist.return_value = [1, 0, 0, 1, 0, 0]
result = worker.lookup(
96,
[f"h{i}".encode() for i in range(12)],
)
assert result == 32
keys = worker.store.batch_is_exist.call_args.args[0]
assert len(keys) == 6
swa_keys = [key for key in keys if "@group:1@" in key]
assert swa_keys == [
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6833",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6837",
"test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@683131",
]
# ---------------------------------------------------------------------------
# register_kv_caches tests
# ---------------------------------------------------------------------------
@@ -172,19 +172,50 @@ class MooncakeStoreCoordinator:
self,
aligned_token_len: int,
num_prompt_tokens: int | None = None,
) -> tuple[list[bool], ...]:
"""Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of
group ``g`` should be written to the store so a future cache hit can
consume it.
) -> tuple[list[bool] | None, ...]:
"""Per-group store masks.
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
written to the store so a future cache hit can consume it. ``None`` is
the all-True sentinel.
Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask``
so the store retains exactly the blocks the local prefix cache would.
"""
return self._reachable_masks(
aligned_token_len,
retention_interval=self.retention_interval,
num_prompt_tokens=num_prompt_tokens,
)
def lookup_mask(
self,
aligned_token_len: int,
) -> tuple[list[bool] | None, ...]:
"""Per-group lookup masks.
``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be
looked up as an aligned hit boundary. ``None`` is the all-True
sentinel.
"""
return self._reachable_masks(
aligned_token_len,
retention_interval=None,
num_prompt_tokens=None,
)
def _reachable_masks(
self,
aligned_token_len: int,
*,
retention_interval: int | None,
num_prompt_tokens: int | None,
) -> tuple[list[bool] | None, ...]:
assert aligned_token_len % self.lcm_block_size == 0, (
f"aligned_token_len ({aligned_token_len}) must be a multiple of "
f"lcm_block_size ({self.lcm_block_size})"
)
masks: list[list[bool]] = []
masks: list[list[bool] | None] = []
for g_idx, g in enumerate(self.kv_cache_groups):
spec = _unwrap_spec(g.kv_cache_spec)
num_chunks = aligned_token_len // spec.block_size
@@ -196,10 +227,12 @@ class MooncakeStoreCoordinator:
alignment_tokens=self.lcm_block_size,
kv_cache_spec=spec,
use_eagle=g_idx in self.eagle_group_ids,
retention_interval=self.retention_interval,
retention_interval=retention_interval,
num_prompt_tokens=num_prompt_tokens,
)
masks.append([True] * num_chunks if mask is None else mask)
if mask is not None:
assert len(mask) == num_chunks
masks.append(mask)
return tuple(masks)
def block_hashes_for_spec(
@@ -548,7 +548,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
for chunk_idx, (start, end, key) in enumerate(
db.process_tokens(token_len, req_meta.block_hashes)
):
if chunk_idx >= len(mask) or not mask[chunk_idx]:
if mask is not None and (
chunk_idx >= len(mask) or not mask[chunk_idx]
):
continue
starts.append(start)
ends.append(end)
@@ -1375,9 +1377,11 @@ class MooncakeStoreWorker:
# candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i].
candidate_keys: list[str] = []
candidate_meta: list[tuple[int, bytes]] = []
lookup_masks = self.coord.lookup_mask(token_len)
tp_count = min(self.tp_size, self.num_kv_head)
for g_idx, db in enumerate(self.token_dbs):
spec_block_size = db.block_size
lookup_mask = lookup_masks[g_idx]
group_hashes = self.coord.block_hashes_for_spec(
block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec
)
@@ -1385,6 +1389,10 @@ class MooncakeStoreWorker:
start_idx = chunk_id * spec_block_size
if start_idx >= token_len:
break
if lookup_mask is not None and (
chunk_id >= len(lookup_mask) or not lookup_mask[chunk_id]
):
continue
for tp in range(tp_count):
for pp in range(self.pp_size):
md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)