(feat): optionally disable lookup on PD decode (#50498)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jeff (Junze) Ma
2026-08-01 09:41:43 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 81a42d37b1
commit 3986b967b2
5 changed files with 67 additions and 3 deletions
@@ -138,9 +138,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
)
assert vllm_config.kv_transfer_config is not None
assert kv_cache_config is not None, "kv_cache_config is required"
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self.kv_role = vllm_config.kv_transfer_config.kv_role
# Capacity-only: contributes its segment to the store pool but transfers
# no KV, so the KV-cache-shape invariants below cannot be reached.
self._capacity_only = self.kv_role == "kv_consumer" and not (
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"enable_lookup", True
)
)
if not self._capacity_only:
self._validate_kv_cache_config(vllm_config, kv_cache_config)
self._kv_cache_config = kv_cache_config
self._kv_cache_events: MooncakeStoreKVEvents | None = None
self.connector_scheduler: MooncakeStoreScheduler | None = None
@@ -60,6 +60,8 @@ class MooncakeStoreScheduler:
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
self.load_async = kvc_extra_config.get("load_async", True)
self.lookup_async = kvc_extra_config.get("lookup_async", False)
# Skips lookup CPU cost on instances that never load KV from the store.
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
self.client = LookupKeyClient(vllm_config)
# Align with the engine's own scheduler_block_size and hash_block_size.
@@ -86,6 +88,9 @@ class MooncakeStoreScheduler:
Returns ``(None, False)`` when an async lookup is still in flight,
signaling the scheduler to retry this request on a later step.
"""
if not self.enable_lookup:
return 0, False
# Fine-grained hits may land on a hash boundary inside a block; without
# partial hits, prefixes shorter than one physical block are skipped.
align = (
@@ -1172,6 +1172,12 @@ class MooncakeStoreWorker:
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"load_async", True
)
# Mirrors MooncakeStoreConnector._capacity_only.
self._capacity_only = self.kv_role == "kv_consumer" and not (
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
"enable_lookup", True
)
)
self.cache_config = vllm_config.cache_config
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
kv_cache_config, vllm_config
@@ -1266,6 +1272,17 @@ class MooncakeStoreWorker:
self.kv_connector_stats = MooncakeStoreConnectorStats()
self._kv_cache_config = kv_cache_config
self.token_dbs: list[ChunkedTokenDatabase] = []
# a capacity-only instance does not need below utils
if self._capacity_only:
logger.info(
"Mooncake store in capacity-only mode: segment mounted "
"(global_segment_size=%d), KV transfer disabled.",
store_config.global_segment_size,
)
return
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
# self.block_size (= scheduler_block_size) so the coordinator's
# ``block_size % hash_block_size == 0`` invariant holds.
@@ -1314,7 +1331,7 @@ class MooncakeStoreWorker:
self._group_tp_replication_factors: tuple[int, ...] = (
self._compute_group_tp_replication_factors()
)
self.token_dbs: list[ChunkedTokenDatabase] = [
self.token_dbs = [
ChunkedTokenDatabase(
dataclasses.replace(
metadata,
@@ -1405,6 +1422,8 @@ class MooncakeStoreWorker:
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
) -> None:
"""Register KV cache tensors and start transfer threads."""
if self._capacity_only:
return
if not kv_caches:
logger.warning("No KV caches to offload.")
return
@@ -1541,6 +1560,9 @@ class MooncakeStoreWorker:
compute is launched on the compute stream) for better
compute-I/O overlap.
"""
if self._capacity_only:
return set(), set()
# Issue async loads
for request in meta.requests:
load_spec = request.load_spec
@@ -1659,6 +1681,9 @@ class MooncakeStoreWorker:
hit covering all ``num_tokens`` is re-derived below the request end so
the last token is recomputed for sampling.
"""
if self._capacity_only:
return 0
token_len = self.coord.align_lookup_length(num_tokens)
if not block_hashes or token_len <= 0:
return 0