From 5fd3b276f8fa34b70d3c83314700f626c66f9a22 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Thu, 18 Jun 2026 05:23:20 -0400 Subject: [PATCH] [Mooncake] Skip KV lookup for non-reachable SWA blocks (#45444) Signed-off-by: wzhao18 --- .../unit/test_mooncake_store_coordinator.py | 20 +++--- .../unit/test_mooncake_store_worker.py | 61 +++++++++++++++++++ .../v1/mooncake/store/coordinator.py | 47 +++++++++++--- .../kv_connector/v1/mooncake/store/worker.py | 10 ++- 4 files changed, 120 insertions(+), 18 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py index 677e4de22b..8d00345157 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_coordinator.py @@ -200,7 +200,7 @@ def test_store_mask_full_attention_all_true(): groups = [KVCacheGroupSpec(["L0"], _full(16))] coord = _make_coord(groups, hash_block_size=16) masks = coord.store_mask(64) - assert masks == ([True, True, True, True],) + assert masks == (None,) def test_store_mask_zero_aligned_returns_empty_per_group(): @@ -210,7 +210,7 @@ def test_store_mask_zero_aligned_returns_empty_per_group(): ] coord = _make_coord(groups, hash_block_size=16) masks = coord.store_mask(0) - assert masks == ([], []) + assert masks == (None, None) def test_store_mask_swa_only_window_around_each_lcm_boundary(): @@ -224,7 +224,7 @@ def test_store_mask_swa_only_window_around_each_lcm_boundary(): coord = _make_coord(groups, hash_block_size=8) masks = coord.store_mask(64) # Full-attn: 2 chunks * 32 tokens. - assert masks[0] == [True, True] + assert masks[0] is None # SWA: 8 chunks * 8 tokens. Only chunks ending at 32 and 64 are stored. assert masks[1] == [False, False, False, True, False, False, False, True] @@ -237,7 +237,7 @@ def test_store_mask_swa_wider_window_covers_more_blocks_per_lcm(): groups = [KVCacheGroupSpec(["L0"], full), KVCacheGroupSpec(["L1"], swa)] coord = _make_coord(groups, hash_block_size=8) masks = coord.store_mask(64) - assert masks[0] == [True, True] + assert masks[0] is None # Boundary at 32: blocks ending in [16, 32) — chunks 2 and 3. # Boundary at 64: chunks 6 and 7. Others stay False. assert masks[1] == [False, False, True, True, False, False, True, True] @@ -265,12 +265,12 @@ def test_store_mask_dsv4_5_groups_full_mla_plus_4_swa(): masks = coord.store_mask(512) # Full-MLA: 2 chunks of 256, both stored. - assert masks[0] == [True, True] + assert masks[0] is None # SWA(64, sw=128): tail = ceil(127/64) = 2; C = 256/64 = 4. # Per-segment template = [F,F,T,T]; tiled twice. assert masks[1] == [False, False, True, True] * 2 # SWA(64, sw=512): tail = 8 >= C = 4 → entire segment True. - assert masks[2] == [True] * 8 + assert masks[2] is None # SWA(4, sw=16): tail = ceil(15/4) = 4; C = 256/4 = 64. # Last 4 of each 64-chunk segment True. assert masks[3] == ([False] * 60 + [True] * 4) * 2 @@ -289,7 +289,7 @@ def test_store_mask_fast_path_all_block_sizes_equal_lcm(): assert coord.lcm_block_size == 64 masks = coord.store_mask(256) # Every block in every group is True — no sub-lcm filtering possible. - assert masks == ([True] * 4, [True] * 4) + assert masks == (None, None) def test_store_mask_fast_path_single_attention_group(): @@ -300,7 +300,7 @@ def test_store_mask_fast_path_single_attention_group(): coord = _make_coord(groups, hash_block_size=16) assert len(coord.attention_groups) == 1 masks = coord.store_mask(64) - assert masks == ([True] * 4, [True] * 4) + assert masks == (None, None) # ----- store_mask with retention_interval (DSV4 sparse SWA checkpointing) ----- @@ -319,7 +319,7 @@ def test_store_mask_dense_default_matches_every_lcm_boundary(): boundary: tokens 32/64/96/128 -> chunks 3/7/11/15.""" coord = _make_coord(_retention_groups(), hash_block_size=8) masks = coord.store_mask(128) - assert masks[0] == [True, True, True, True] + assert masks[0] is None assert masks[1] == [i % 4 == 3 for i in range(16)] @@ -329,7 +329,7 @@ def test_store_mask_retention_interval_sparsifies_swa_tails(): boundaries at 32 and 96.""" coord = _make_coord(_retention_groups(), hash_block_size=8, retention_interval=64) masks = coord.store_mask(128) - assert masks[0] == [True, True, True, True] # full attn unaffected + assert masks[0] is None # full attn unaffected assert masks[1] == [i in (7, 15) for i in range(16)] diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 78e0f3cacb..aa5d7d1ff3 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -1233,6 +1233,67 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present(): assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64 +def test_lookup_checks_all_potential_swa_hit_boundaries(): + """Lookup should skip SWA chunks that can never validate a hit, but still + check earlier aligned boundaries when sparse retention stores only the + current request's replay boundary. + """ + from vllm.v1.kv_cache_interface import ( + FullAttentionSpec, + KVCacheGroupSpec, + SlidingWindowSpec, + ) + + worker = _make_bare_worker(block_size=8) + full = FullAttentionSpec(block_size=32, num_kv_heads=8, head_size=64, dtype=None) + swa = SlidingWindowSpec( + block_size=8, num_kv_heads=8, head_size=64, dtype=None, sliding_window=8 + ) + worker._kv_cache_groups = [ + KVCacheGroupSpec(["full"], full), + KVCacheGroupSpec(["swa"], swa), + ] + worker.token_dbs = [ + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=0), + block_size=32, + hash_block_size=8, + ), + ChunkedTokenDatabase( + KeyMetadata("test-model", 0, 0, 0, 0, group_id=1), + block_size=8, + hash_block_size=8, + ), + ] + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=32, + hash_block_size=8, + retention_interval=0, + ) + # Candidate order: 3 full-attention chunks, then SWA chunks 3, 7, 11. + # Only the first full chunk and the SWA chunk ending at token 32 exist, so + # lookup should recover a 32-token external prefix hit. A sparse + # prompt-specific store mask for num_prompt_tokens=96 would only check SWA + # chunk 7 and miss this earlier reusable prefix. + worker.store.batch_is_exist.return_value = [1, 0, 0, 1, 0, 0] + + result = worker.lookup( + 96, + [f"h{i}".encode() for i in range(12)], + ) + + assert result == 32 + keys = worker.store.batch_is_exist.call_args.args[0] + assert len(keys) == 6 + swa_keys = [key for key in keys if "@group:1@" in key] + assert swa_keys == [ + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6833", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@6837", + "test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:1@683131", + ] + + # --------------------------------------------------------------------------- # register_kv_caches tests # --------------------------------------------------------------------------- diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index 227575c926..b1513e7269 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -172,19 +172,50 @@ class MooncakeStoreCoordinator: self, aligned_token_len: int, num_prompt_tokens: int | None = None, - ) -> tuple[list[bool], ...]: - """Per-group store masks: ``mask[g][i]`` is True iff chunk ``i`` of - group ``g`` should be written to the store so a future cache hit can - consume it. + ) -> tuple[list[bool] | None, ...]: + """Per-group store masks. + + ``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be + written to the store so a future cache hit can consume it. ``None`` is + the all-True sentinel. Reuses the engine's ``SingleTypeKVCacheManager.reachable_block_mask`` so the store retains exactly the blocks the local prefix cache would. """ + return self._reachable_masks( + aligned_token_len, + retention_interval=self.retention_interval, + num_prompt_tokens=num_prompt_tokens, + ) + + def lookup_mask( + self, + aligned_token_len: int, + ) -> tuple[list[bool] | None, ...]: + """Per-group lookup masks. + + ``mask[g][i]`` is True iff chunk ``i`` of group ``g`` should be + looked up as an aligned hit boundary. ``None`` is the all-True + sentinel. + """ + return self._reachable_masks( + aligned_token_len, + retention_interval=None, + num_prompt_tokens=None, + ) + + def _reachable_masks( + self, + aligned_token_len: int, + *, + retention_interval: int | None, + num_prompt_tokens: int | None, + ) -> tuple[list[bool] | None, ...]: assert aligned_token_len % self.lcm_block_size == 0, ( f"aligned_token_len ({aligned_token_len}) must be a multiple of " f"lcm_block_size ({self.lcm_block_size})" ) - masks: list[list[bool]] = [] + masks: list[list[bool] | None] = [] for g_idx, g in enumerate(self.kv_cache_groups): spec = _unwrap_spec(g.kv_cache_spec) num_chunks = aligned_token_len // spec.block_size @@ -196,10 +227,12 @@ class MooncakeStoreCoordinator: alignment_tokens=self.lcm_block_size, kv_cache_spec=spec, use_eagle=g_idx in self.eagle_group_ids, - retention_interval=self.retention_interval, + retention_interval=retention_interval, num_prompt_tokens=num_prompt_tokens, ) - masks.append([True] * num_chunks if mask is None else mask) + if mask is not None: + assert len(mask) == num_chunks + masks.append(mask) return tuple(masks) def block_hashes_for_spec( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 4160d426fe..f5a55b54c7 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -548,7 +548,9 @@ class KVCacheStoreSendingThread(KVTransferThread): for chunk_idx, (start, end, key) in enumerate( db.process_tokens(token_len, req_meta.block_hashes) ): - if chunk_idx >= len(mask) or not mask[chunk_idx]: + if mask is not None and ( + chunk_idx >= len(mask) or not mask[chunk_idx] + ): continue starts.append(start) ends.append(end) @@ -1375,9 +1377,11 @@ class MooncakeStoreWorker: # candidate_meta[i] is the (group_id, hash_bytes) for candidate_keys[i]. candidate_keys: list[str] = [] candidate_meta: list[tuple[int, bytes]] = [] + lookup_masks = self.coord.lookup_mask(token_len) tp_count = min(self.tp_size, self.num_kv_head) for g_idx, db in enumerate(self.token_dbs): spec_block_size = db.block_size + lookup_mask = lookup_masks[g_idx] group_hashes = self.coord.block_hashes_for_spec( block_hashes, self._kv_cache_groups[g_idx].kv_cache_spec ) @@ -1385,6 +1389,10 @@ class MooncakeStoreWorker: start_idx = chunk_id * spec_block_size if start_idx >= token_len: break + if lookup_mask is not None and ( + chunk_id >= len(lookup_mask) or not lookup_mask[chunk_id] + ): + continue for tp in range(tp_count): for pp in range(self.pp_size): md = dataclasses.replace(db.metadata, tp_rank=tp, pp_rank=pp)