From a76df87db89bb87213f60cff817d5789796ee158 Mon Sep 17 00:00:00 2001 From: Dao007forever Date: Thu, 23 Jul 2026 03:12:28 -0700 Subject: [PATCH] [MooncakeStore] Re-derive full external hits on stored boundaries (#49481) Signed-off-by: Dao Le Signed-off-by: Yifan Qiao Co-authored-by: Yifan Qiao Co-authored-by: Claude --- .../unit/test_mooncake_store_connector.py | 6 +- .../unit/test_mooncake_store_hma_e2e.py | 12 +++- .../unit/test_mooncake_store_scheduler.py | 17 +++--- .../unit/test_mooncake_store_worker.py | 59 ++++++++++++++++++- .../v1/mooncake/store/coordinator.py | 3 + .../v1/mooncake/store/protocol.py | 3 +- .../v1/mooncake/store/scheduler.py | 14 +---- .../kv_connector/v1/mooncake/store/worker.py | 34 ++++++++--- 8 files changed, 110 insertions(+), 38 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py index 951b447fd6..4dcddcaa30 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_connector.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_connector.py @@ -410,7 +410,7 @@ def test_lookup_key_client_lookup_prepends_typed_tag(): # Blocking lookup (non_block defaults to False) runs on the executor and # returns the resolved hit length. - assert client.lookup("req0", token_len=128, block_hashes=[]) == 5 + assert client.lookup("req0", num_tokens=128, block_hashes=[]) == 5 sent_frames = fake_socket.send_multipart.call_args[0][0] assert sent_frames[0] == protocol.LOOKUP_MSG @@ -439,11 +439,11 @@ def test_lookup_key_client_reset_uses_typed_protocol(): assert client.reset() is False -def _poll_lookup(client, req_id, token_len=128, block_hashes=(), timeout=5.0): +def _poll_lookup(client, req_id, num_tokens=128, block_hashes=(), timeout=5.0): """Drive non-blocking lookup until the executor completes it.""" deadline = time.monotonic() + timeout while time.monotonic() < deadline: - result = client.lookup(req_id, token_len, list(block_hashes), non_block=True) + result = client.lookup(req_id, num_tokens, list(block_hashes), non_block=True) if result is not None: return result time.sleep(0.005) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index f09e0a2472..46a70654b1 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -240,7 +240,10 @@ def test_e2e_swa_plus_full_save_then_lookup_hits(): worker.store = store # Both groups stored all 4 blocks -> full hit. - assert worker.lookup(token_len=64, block_hashes=hs) == 64 + assert worker.lookup(num_tokens=65, block_hashes=hs) == 64 + # Exact-multiple prompt: the full hit is re-derived one block lower, + # where both groups' stored blocks still cover the SWA window. + assert worker.lookup(num_tokens=64, block_hashes=hs) == 48 # Evict SWA's first two blocks (outside its window of 32 tokens = 2 blocks). swa_keys_outside_window = [ @@ -253,7 +256,12 @@ def test_e2e_swa_plus_full_save_then_lookup_hits(): # SWA window=32 -> only last 2 blocks must be present in SWA group. # Full has all 4. Coordinator should still return 64. - assert worker.lookup(token_len=64, block_hashes=hs) == 64 + assert worker.lookup(num_tokens=65, block_hashes=hs) == 64 + # Exact-multiple prompt after eviction: the boundary one block lower + # needs SWA block 1, which is gone — no usable stored boundary remains + # (the pre-fix arithmetic clamp would have returned 48 and livelocked + # on load failure -> recompute -> same lookup). + assert worker.lookup(num_tokens=64, block_hashes=hs) == 0 def test_recv_skips_swa_blocks_before_window(): diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 7e29196298..e7a0cbc1a7 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -473,27 +473,25 @@ def test_from_request_tracker_no_load_saves_normally(): class _StubLookupClient: def __init__(self, hit_tokens: int) -> None: self._hit_tokens = hit_tokens + self.num_tokens: list[int] = [] def lookup( self, req_id: str, - token_len: int, + num_tokens: int, block_hashes: list[bytes], non_block: bool = False, ) -> int: + self.num_tokens.append(num_tokens) return self._hit_tokens def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned(): - # When the external store hits the entire prompt, scheduler must leave at - # least one token uncomputed for sampling but stay on a block boundary. - # Otherwise the recv-side load mask floors token_len to - # (num_tokens-1)//block_size, the tail partial chunk is dropped, and -- if - # the local cache covers the aligned prefix -- key_list ends up empty - # (ZeroDivisionError in the recv thread's `tp_rank % len(key_list)`). + # The worker re-derives a full external hit below the request end on an + # existing boundary, so the scheduler receives the usable aligned hit. scheduler = _make_bare_scheduler() scheduler.load_async = True - scheduler.client = _StubLookupClient(hit_tokens=48) # full hit on 48-token prompt + scheduler.client = _StubLookupClient(hit_tokens=32) request = SimpleNamespace( request_id="req-0", @@ -510,6 +508,7 @@ def test_full_external_hit_keeps_kvpool_cached_tokens_block_aligned(): assert need_to_allocate == 16 assert load_async is True load_spec = scheduler.load_specs["req-0"] + assert scheduler.client.num_tokens == [48] assert load_spec.vllm_cached_tokens == 16 assert load_spec.kvpool_cached_tokens == 32 assert load_spec.kvpool_cached_tokens % 16 == 0 @@ -522,7 +521,7 @@ def test_full_external_hit_with_full_local_hit_skips_load(): # into any block-aligned key. scheduler = _make_bare_scheduler() scheduler.load_async = True - scheduler.client = _StubLookupClient(hit_tokens=48) + scheduler.client = _StubLookupClient(hit_tokens=32) request = SimpleNamespace( request_id="req-0", diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 4d9e974b41..ec47bfa238 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -1737,6 +1737,61 @@ def test_lookup_partial_prefix_returns_first_hit_length(): assert worker.lookup(48, [b"a0", b"a1", b"a2"]) == 32 +def test_lookup_full_hit_reuses_existing_boundary(): + """A full hit is re-derived below the request end without another RPC.""" + worker = _make_bare_worker(block_size=16) + worker.store.batch_is_exist.return_value = [1, 1] + + assert worker.lookup(32, [b"h0", b"h1"]) == 16 + assert worker.store.batch_is_exist.call_count == 1 + + +def test_lookup_full_hit_with_eagle_pops_once_not_twice(): + """Eagle already leaves the last block for the drafter, so a + full-prompt re-derivation must never fire for eagle-governed hits: + firing would anchor the search one block lower and pop a second + block, regressing the hit by an extra producer boundary.""" + worker = _make_bare_worker(block_size=16) + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=16, + hash_block_size=16, + use_eagle=True, + ) + worker.store.batch_is_exist.return_value = [1, 1, 1, 1] + + # 64-token exact-multiple prompt, all 4 blocks stored: one eagle pop + # gives 48; a spurious re-derivation (anchored at 48) would pop again + # and return 32. + assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 48 + assert worker.store.batch_is_exist.call_count == 1 + + +def test_lookup_full_hit_swa_degrades_when_no_stored_boundary_is_usable(): + """The motivating livelock: the producer of a 64-token prompt stored + only its SWA tail window (blocks 2-3). The old arithmetic clamp turned + the full hit into 48, whose SWA window needs the never-written block 1, + so every load failed and the recompute re-entered the same lookup. The + re-derivation must report that no stored boundary below the request end + is usable.""" + from vllm.v1.kv_cache_interface import KVCacheGroupSpec, SlidingWindowSpec + + worker = _make_bare_worker(block_size=16) + swa = SlidingWindowSpec( + block_size=16, num_kv_heads=8, head_size=64, dtype=None, sliding_window=32 + ) + worker._kv_cache_groups = [KVCacheGroupSpec(["layer0"], swa)] + worker.coord = mooncake_store_worker.MooncakeStoreCoordinator( + worker._kv_cache_groups, + scheduler_block_size=worker.hash_block_size, + hash_block_size=worker.hash_block_size, + ) + worker.store.batch_is_exist.return_value = [0, 0, 1, 1] + + assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 0 + assert worker.store.batch_is_exist.call_count == 1 + + def test_lookup_swa_single_group_returns_full_when_tail_window_present(): """Single-SWA, sliding_window=32 (= 2 blocks): producer stored only the tail. Coordinator-driven lookup returns full prefix even though the @@ -1754,7 +1809,7 @@ def test_lookup_swa_single_group_returns_full_when_tail_window_present(): hash_block_size=worker.hash_block_size, ) worker.store.batch_is_exist.return_value = [0, 0, 1, 1] - assert worker.lookup(64, [b"h0", b"h1", b"h2", b"h3"]) == 64 + assert worker.lookup(65, [b"h0", b"h1", b"h2", b"h3"]) == 64 def test_lookup_checks_all_potential_swa_hit_boundaries(): @@ -2219,7 +2274,7 @@ def test_lookup_records_mooncake_metrics(): worker = _make_bare_worker() worker.store.batch_is_exist.return_value = [1, 1] - result = worker.lookup(32, [b"a0", b"a1"]) + result = worker.lookup(33, [b"a0", b"a1"]) stats = worker.get_kv_connector_stats() assert result == 32 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py index 92283d3a53..db34079ef8 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/coordinator.py @@ -93,6 +93,9 @@ class MooncakeStoreCoordinator: self.eagle_group_ids = set(range(len(kv_cache_groups))) self._verify_and_split_kv_cache_groups() + def align_lookup_length(self, length: int) -> int: + return length // self.lcm_block_size * self.lcm_block_size + def _verify_and_split_kv_cache_groups(self) -> None: """Mirrors KVCacheCoordinator.verify_and_split_kv_cache_groups but dispatches via spec_manager_map (we don't allocate managers). diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py index fc91b0aeeb..eb6c65afa3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/protocol.py @@ -10,7 +10,8 @@ Wire format (REQ/REP over IPC): Request: [msg_type: bytes] [payload_frames...] msg_type == LOOKUP_MSG: - frame 1: token_len (u32 big-endian, 4 bytes) + frame 1: num_tokens (u32 big-endian, 4 bytes); the worker derives + the aligned lookup length frame 2: hash_len (u16 big-endian, 2 bytes) — byte length of each fixed-size block hash (0 when there are no hashes) frame 3: raw block hashes concatenated back-to-back (each hash_len diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 58dfd5e428..2fef49a307 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -80,14 +80,12 @@ class MooncakeStoreScheduler: Returns ``(None, False)`` when an async lookup is still in flight, signaling the scheduler to retry this request on a later step. """ - # Look up against the full prefill range, not just the prompt. - token_len = request.num_tokens // self._block_size * self._block_size - if token_len < self._block_size: + if request.num_tokens < self._block_size: return 0, False num_external_hit_tokens = self.client.lookup( request.request_id, - token_len, + request.num_tokens, request.block_hashes, non_block=self.lookup_async, ) @@ -95,14 +93,6 @@ class MooncakeStoreScheduler: # Lookup not ready yet; scheduler will retry on a later step. return None, False - if num_external_hit_tokens == request.num_tokens: - # Leave a sub-block tail uncomputed for sampling, on a block - # boundary so the recv-side load mask covers every yielded chunk. - num_external_hit_tokens = max( - 0, - (request.num_tokens - 1) // self._block_size * self._block_size, - ) - if num_external_hit_tokens < num_computed_tokens: need_to_allocate = 0 else: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 0ca73d4211..03b2589c07 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1467,11 +1467,14 @@ class MooncakeStoreWorker: return finished_sending - def lookup(self, token_len: int, block_hashes: Sequence[BlockHash]) -> int: + def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int: """Check how many prefix tokens exist in the store. - Checks across all rank-specific key namespaces that may be loaded. + Checks across all rank-specific key namespaces that may be loaded. A + hit covering all ``num_tokens`` is re-derived below the request end so + the last token is recomputed for sampling. """ + token_len = self.coord.align_lookup_length(num_tokens) if not block_hashes or token_len <= 0: return 0 @@ -1535,11 +1538,24 @@ class MooncakeStoreWorker: ) } + cached_block_pool = ExternalCachedBlockPool( + self.hash_block_size, + exists_set, + ) _masks, hit_length = self.coord.find_longest_cache_hit( block_hashes, token_len, - ExternalCachedBlockPool(self.hash_block_size, exists_set), + cached_block_pool, ) + if hit_length >= num_tokens: + usable_length = self.coord.align_lookup_length(num_tokens - 1) + if usable_length <= 0: + return 0 + _masks, hit_length = self.coord.find_longest_cache_hit( + block_hashes, + usable_length, + cached_block_pool, + ) return hit_length def get_kv_events(self) -> list[BlockStored]: @@ -1605,11 +1621,11 @@ class LookupKeyServer: msg_type = bytes(all_frames[0]) if msg_type == LOOKUP_MSG: - token_len = int.from_bytes(all_frames[1], byteorder="big") + num_tokens = int.from_bytes(all_frames[1], byteorder="big") hash_len = int.from_bytes(all_frames[2], byteorder="big") blob = all_frames[3].buffer block_hashes = BlobBlockHashes(blob, hash_len) - result = self.store_worker.lookup(token_len, block_hashes) + result = self.store_worker.lookup(num_tokens, block_hashes) self.socket.send(result.to_bytes(4, "big")) elif msg_type == RESET_MSG: @@ -1672,11 +1688,11 @@ class LookupKeyClient: ) self.futures: dict[str, Future[int]] = {} - def _lookup(self, token_len: int, block_hashes: list[BlockHash]) -> int: + def _lookup(self, num_tokens: int, block_hashes: list[BlockHash]) -> int: hash_len = len(block_hashes[0]) if block_hashes else 0 all_frames = ( LOOKUP_MSG, - token_len.to_bytes(4, byteorder="big"), + num_tokens.to_bytes(4, byteorder="big"), hash_len.to_bytes(2, byteorder="big"), b"".join(block_hashes), ) @@ -1687,7 +1703,7 @@ class LookupKeyClient: def lookup( self, req_id: str, - token_len: int, + num_tokens: int, block_hashes: list[BlockHash], non_block: bool = False, ) -> int | None: @@ -1695,7 +1711,7 @@ class LookupKeyClient: so the caller retries on a later step.""" future = self.futures.get(req_id) if future is None: - future = self.executor.submit(self._lookup, token_len, list(block_hashes)) + future = self.executor.submit(self._lookup, num_tokens, list(block_hashes)) self.futures[req_id] = future if non_block and not future.done(): return None