(feat): optionally disable lookup on PD decode (#50498)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jeff (Junze) Ma
2026-08-01 09:41:43 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 81a42d37b1
commit 3986b967b2
5 changed files with 67 additions and 3 deletions
@@ -19,6 +19,7 @@ def _make_bare_scheduler(
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.lookup_async = False
scheduler.enable_lookup = True
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
@@ -636,6 +637,30 @@ def test_sub_block_prompt_not_looked_up_without_fine_grained():
assert "req-0" not in scheduler.load_specs
def test_disabled_lookup_reports_no_hit_without_querying_client():
# With enable_lookup=False the connector reports no external hit without
# consulting the lookup client, so admission is never deferred on a store
# lookup. Used by instances that only contribute store capacity.
scheduler = _make_bare_scheduler()
scheduler.enable_lookup = False
scheduler.client = _StubLookupClient(hit_tokens=32)
request = SimpleNamespace(
request_id="req-0",
num_tokens=48,
block_hashes=[b"h0", b"h1", b"h2"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=0
)
assert need_to_allocate == 0
assert load_async is False
assert scheduler.client.num_tokens == []
assert scheduler.load_specs == {}
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
@@ -1746,6 +1746,7 @@ def _make_bare_worker(
worker.store = MagicMock()
worker.store.register_buffer.return_value = 0
worker.kv_role = kv_role
worker._capacity_only = False
worker.block_size = block_size
worker.tp_rank = 0
worker.enable_kv_events = False
@@ -138,9 +138,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self.kv_role = vllm_config.kv_transfer_config.kv_role
# Capacity-only: contributes its segment to the store pool but transfers
# no KV, so the KV-cache-shape invariants below cannot be reached.
self._capacity_only = self.kv_role == "kv_consumer" and not (
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"enable_lookup", True
)
)
if not self._capacity_only:
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self._kv_cache_events: MooncakeStoreKVEvents | None = None
self.connector_scheduler: MooncakeStoreScheduler | None = None
@@ -60,6 +60,8 @@ class MooncakeStoreScheduler:
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.load_async = kvc_extra_config.get("load_async", True)
self.lookup_async = kvc_extra_config.get("lookup_async", False)
# Skips lookup CPU cost on instances that never load KV from the store.
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
self.client = LookupKeyClient(vllm_config)
# Align with the engine's own scheduler_block_size and hash_block_size.
@@ -86,6 +88,9 @@ class MooncakeStoreScheduler:
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if not self.enable_lookup:
return 0, False
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
@@ -1172,6 +1172,12 @@ class MooncakeStoreWorker:
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"load_async", True
)
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = self.kv_role == "kv_consumer" and not (
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"enable_lookup", True
)
)
self.cache_config = vllm_config.cache_config
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
@@ -1266,6 +1272,17 @@ class MooncakeStoreWorker:
self.kv_connector_stats = MooncakeStoreConnectorStats()
self._kv_cache_config = kv_cache_config
self.token_dbs: list[ChunkedTokenDatabase] = []
# a capacity-only instance does not need below utils
if self._capacity_only:
logger.info(
"Mooncake store in capacity-only mode: segment mounted "
"(global_segment_size=%d), KV transfer disabled.",
store_config.global_segment_size,
)
return
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
@@ -1314,7 +1331,7 @@ class MooncakeStoreWorker:
self._group_tp_replication_factors: tuple[int, ...] = (
self._compute_group_tp_replication_factors()
)
self.token_dbs: list[ChunkedTokenDatabase] = [
self.token_dbs = [
ChunkedTokenDatabase(
dataclasses.replace(
metadata,
@@ -1405,6 +1422,8 @@ class MooncakeStoreWorker:
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
) -> None:
"""Register KV cache tensors and start transfer threads."""
if self._capacity_only:
return
if not kv_caches:
logger.warning("No KV caches to offload.")
return
@@ -1541,6 +1560,9 @@ class MooncakeStoreWorker:
compute is launched on the compute stream) for better
compute-I/O overlap.
"""
if self._capacity_only:
return set(), set()
# Issue async loads
for request in meta.requests:
load_spec = request.load_spec
@@ -1659,6 +1681,9 @@ class MooncakeStoreWorker:
hit covering all ``num_tokens`` is re-derived below the request end so
the last token is recomputed for sampling.
"""
if self._capacity_only:
return 0
token_len = self.coord.align_lookup_length(num_tokens)
if not block_hashes or token_len <= 0:
return 0