From 3986b967b249468fc1bd052f89a1eeeada6377b4 Mon Sep 17 00:00:00 2001 From: "Jeff (Junze) Ma" <93145857+majunze2001@users.noreply.github.com> Date: Sat, 1 Aug 2026 02:41:43 -0700 Subject: [PATCH] (feat): optionally disable lookup on PD decode (#50498) Signed-off-by: Jeff Ma Co-authored-by: Yifan Qiao --- .../unit/test_mooncake_store_scheduler.py | 25 +++++++++++++++++ .../unit/test_mooncake_store_worker.py | 1 + .../v1/mooncake/store/connector.py | 12 +++++++-- .../v1/mooncake/store/scheduler.py | 5 ++++ .../kv_connector/v1/mooncake/store/worker.py | 27 ++++++++++++++++++- 5 files changed, 67 insertions(+), 3 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 2709b92e52..13960c4034 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -19,6 +19,7 @@ def _make_bare_scheduler( scheduler = object.__new__(MooncakeStoreScheduler) scheduler.kv_role = "kv_both" scheduler.lookup_async = False + scheduler.enable_lookup = True scheduler._block_size = 16 scheduler._hash_block_size = hash_block_size scheduler.enable_partial_hash_hits = enable_partial_hash_hits @@ -636,6 +637,30 @@ def test_sub_block_prompt_not_looked_up_without_fine_grained(): assert "req-0" not in scheduler.load_specs +def test_disabled_lookup_reports_no_hit_without_querying_client(): + # With enable_lookup=False the connector reports no external hit without + # consulting the lookup client, so admission is never deferred on a store + # lookup. Used by instances that only contribute store capacity. + scheduler = _make_bare_scheduler() + scheduler.enable_lookup = False + scheduler.client = _StubLookupClient(hit_tokens=32) + + request = SimpleNamespace( + request_id="req-0", + num_tokens=48, + block_hashes=[b"h0", b"h1", b"h2"], + ) + + need_to_allocate, load_async = scheduler.get_num_new_matched_tokens( + request, num_computed_tokens=0 + ) + + assert need_to_allocate == 0 + assert load_async is False + assert scheduler.client.num_tokens == [] + assert scheduler.load_specs == {} + + def test_pending_partial_tail_emits_offload_only_reqmeta(): # A sub-block prompt never produces a block-aligned save, so the partial- # tail offload arriving this step is emitted as an offload-only ReqMeta diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 54553cd49b..0403affea3 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -1746,6 +1746,7 @@ def _make_bare_worker( worker.store = MagicMock() worker.store.register_buffer.return_value = 0 worker.kv_role = kv_role + worker._capacity_only = False worker.block_size = block_size worker.tp_rank = 0 worker.enable_kv_events = False diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index bf6038a897..ca32c3345c 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -138,9 +138,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): ) assert vllm_config.kv_transfer_config is not None assert kv_cache_config is not None, "kv_cache_config is required" - self._validate_kv_cache_config(vllm_config, kv_cache_config) - self._kv_cache_config = kv_cache_config self.kv_role = vllm_config.kv_transfer_config.kv_role + # Capacity-only: contributes its segment to the store pool but transfers + # no KV, so the KV-cache-shape invariants below cannot be reached. + self._capacity_only = self.kv_role == "kv_consumer" and not ( + vllm_config.kv_transfer_config.kv_connector_extra_config.get( + "enable_lookup", True + ) + ) + if not self._capacity_only: + self._validate_kv_cache_config(vllm_config, kv_cache_config) + self._kv_cache_config = kv_cache_config self._kv_cache_events: MooncakeStoreKVEvents | None = None self.connector_scheduler: MooncakeStoreScheduler | None = None diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index a14d02927f..42c6f3fa99 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -60,6 +60,8 @@ class MooncakeStoreScheduler: kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config self.load_async = kvc_extra_config.get("load_async", True) self.lookup_async = kvc_extra_config.get("lookup_async", False) + # Skips lookup CPU cost on instances that never load KV from the store. + self.enable_lookup = kvc_extra_config.get("enable_lookup", True) self.client = LookupKeyClient(vllm_config) # Align with the engine's own scheduler_block_size and hash_block_size. @@ -86,6 +88,9 @@ class MooncakeStoreScheduler: Returns ``(None, False)`` when an async lookup is still in flight, signaling the scheduler to retry this request on a later step. """ + if not self.enable_lookup: + return 0, False + # Fine-grained hits may land on a hash boundary inside a block; without # partial hits, prefixes shorter than one physical block are skipped. align = ( diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 0fd85eb57d..c62eaefc35 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -1172,6 +1172,12 @@ class MooncakeStoreWorker: self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get( "load_async", True ) + # Mirrors MooncakeStoreConnector._capacity_only. + self._capacity_only = self.kv_role == "kv_consumer" and not ( + vllm_config.kv_transfer_config.kv_connector_extra_config.get( + "enable_lookup", True + ) + ) self.cache_config = vllm_config.cache_config self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes( kv_cache_config, vllm_config @@ -1266,6 +1272,17 @@ class MooncakeStoreWorker: self.kv_connector_stats = MooncakeStoreConnectorStats() self._kv_cache_config = kv_cache_config + self.token_dbs: list[ChunkedTokenDatabase] = [] + + # a capacity-only instance does not need below utils + if self._capacity_only: + logger.info( + "Mooncake store in capacity-only mode: segment mounted " + "(global_segment_size=%d), KV transfer disabled.", + store_config.global_segment_size, + ) + return + # Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to # self.block_size (= scheduler_block_size) so the coordinator's # ``block_size % hash_block_size == 0`` invariant holds. @@ -1314,7 +1331,7 @@ class MooncakeStoreWorker: self._group_tp_replication_factors: tuple[int, ...] = ( self._compute_group_tp_replication_factors() ) - self.token_dbs: list[ChunkedTokenDatabase] = [ + self.token_dbs = [ ChunkedTokenDatabase( dataclasses.replace( metadata, @@ -1405,6 +1422,8 @@ class MooncakeStoreWorker: kv_caches: dict[str, torch.Tensor | list[torch.Tensor]], ) -> None: """Register KV cache tensors and start transfer threads.""" + if self._capacity_only: + return if not kv_caches: logger.warning("No KV caches to offload.") return @@ -1541,6 +1560,9 @@ class MooncakeStoreWorker: compute is launched on the compute stream) for better compute-I/O overlap. """ + if self._capacity_only: + return set(), set() + # Issue async loads for request in meta.requests: load_spec = request.load_spec @@ -1659,6 +1681,9 @@ class MooncakeStoreWorker: hit covering all ``num_tokens`` is re-derived below the request end so the last token is recomputed for sampling. """ + if self._capacity_only: + return 0 + token_len = self.coord.align_lookup_length(num_tokens) if not block_hashes or token_len <= 0: return 0