(feat): optionally disable lookup on PD decode (#50498)
Signed-off-by: Jeff Ma <jeffjma@umich.edu> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
81a42d37b1
commit
3986b967b2
@@ -19,6 +19,7 @@ def _make_bare_scheduler(
|
||||
scheduler = object.__new__(MooncakeStoreScheduler)
|
||||
scheduler.kv_role = "kv_both"
|
||||
scheduler.lookup_async = False
|
||||
scheduler.enable_lookup = True
|
||||
scheduler._block_size = 16
|
||||
scheduler._hash_block_size = hash_block_size
|
||||
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
|
||||
@@ -636,6 +637,30 @@ def test_sub_block_prompt_not_looked_up_without_fine_grained():
|
||||
assert "req-0" not in scheduler.load_specs
|
||||
|
||||
|
||||
def test_disabled_lookup_reports_no_hit_without_querying_client():
|
||||
# With enable_lookup=False the connector reports no external hit without
|
||||
# consulting the lookup client, so admission is never deferred on a store
|
||||
# lookup. Used by instances that only contribute store capacity.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler.enable_lookup = False
|
||||
scheduler.client = _StubLookupClient(hit_tokens=32)
|
||||
|
||||
request = SimpleNamespace(
|
||||
request_id="req-0",
|
||||
num_tokens=48,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
)
|
||||
|
||||
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
|
||||
request, num_computed_tokens=0
|
||||
)
|
||||
|
||||
assert need_to_allocate == 0
|
||||
assert load_async is False
|
||||
assert scheduler.client.num_tokens == []
|
||||
assert scheduler.load_specs == {}
|
||||
|
||||
|
||||
def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
# A sub-block prompt never produces a block-aligned save, so the partial-
|
||||
# tail offload arriving this step is emitted as an offload-only ReqMeta
|
||||
|
||||
@@ -1746,6 +1746,7 @@ def _make_bare_worker(
|
||||
worker.store = MagicMock()
|
||||
worker.store.register_buffer.return_value = 0
|
||||
worker.kv_role = kv_role
|
||||
worker._capacity_only = False
|
||||
worker.block_size = block_size
|
||||
worker.tp_rank = 0
|
||||
worker.enable_kv_events = False
|
||||
|
||||
@@ -138,9 +138,17 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
)
|
||||
assert vllm_config.kv_transfer_config is not None
|
||||
assert kv_cache_config is not None, "kv_cache_config is required"
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self.kv_role = vllm_config.kv_transfer_config.kv_role
|
||||
# Capacity-only: contributes its segment to the store pool but transfers
|
||||
# no KV, so the KV-cache-shape invariants below cannot be reached.
|
||||
self._capacity_only = self.kv_role == "kv_consumer" and not (
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"enable_lookup", True
|
||||
)
|
||||
)
|
||||
if not self._capacity_only:
|
||||
self._validate_kv_cache_config(vllm_config, kv_cache_config)
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self._kv_cache_events: MooncakeStoreKVEvents | None = None
|
||||
|
||||
self.connector_scheduler: MooncakeStoreScheduler | None = None
|
||||
|
||||
@@ -60,6 +60,8 @@ class MooncakeStoreScheduler:
|
||||
kvc_extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config
|
||||
self.load_async = kvc_extra_config.get("load_async", True)
|
||||
self.lookup_async = kvc_extra_config.get("lookup_async", False)
|
||||
# Skips lookup CPU cost on instances that never load KV from the store.
|
||||
self.enable_lookup = kvc_extra_config.get("enable_lookup", True)
|
||||
self.client = LookupKeyClient(vllm_config)
|
||||
|
||||
# Align with the engine's own scheduler_block_size and hash_block_size.
|
||||
@@ -86,6 +88,9 @@ class MooncakeStoreScheduler:
|
||||
Returns ``(None, False)`` when an async lookup is still in flight,
|
||||
signaling the scheduler to retry this request on a later step.
|
||||
"""
|
||||
if not self.enable_lookup:
|
||||
return 0, False
|
||||
|
||||
# Fine-grained hits may land on a hash boundary inside a block; without
|
||||
# partial hits, prefixes shorter than one physical block are skipped.
|
||||
align = (
|
||||
|
||||
@@ -1172,6 +1172,12 @@ class MooncakeStoreWorker:
|
||||
self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"load_async", True
|
||||
)
|
||||
# Mirrors MooncakeStoreConnector._capacity_only.
|
||||
self._capacity_only = self.kv_role == "kv_consumer" and not (
|
||||
vllm_config.kv_transfer_config.kv_connector_extra_config.get(
|
||||
"enable_lookup", True
|
||||
)
|
||||
)
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes(
|
||||
kv_cache_config, vllm_config
|
||||
@@ -1266,6 +1272,17 @@ class MooncakeStoreWorker:
|
||||
self.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
|
||||
self._kv_cache_config = kv_cache_config
|
||||
self.token_dbs: list[ChunkedTokenDatabase] = []
|
||||
|
||||
# a capacity-only instance does not need below utils
|
||||
if self._capacity_only:
|
||||
logger.info(
|
||||
"Mooncake store in capacity-only mode: segment mounted "
|
||||
"(global_segment_size=%d), KV transfer disabled.",
|
||||
store_config.global_segment_size,
|
||||
)
|
||||
return
|
||||
|
||||
# Single-group + PCP/DCP > 1: scale the lone group's spec.block_size to
|
||||
# self.block_size (= scheduler_block_size) so the coordinator's
|
||||
# ``block_size % hash_block_size == 0`` invariant holds.
|
||||
@@ -1314,7 +1331,7 @@ class MooncakeStoreWorker:
|
||||
self._group_tp_replication_factors: tuple[int, ...] = (
|
||||
self._compute_group_tp_replication_factors()
|
||||
)
|
||||
self.token_dbs: list[ChunkedTokenDatabase] = [
|
||||
self.token_dbs = [
|
||||
ChunkedTokenDatabase(
|
||||
dataclasses.replace(
|
||||
metadata,
|
||||
@@ -1405,6 +1422,8 @@ class MooncakeStoreWorker:
|
||||
kv_caches: dict[str, torch.Tensor | list[torch.Tensor]],
|
||||
) -> None:
|
||||
"""Register KV cache tensors and start transfer threads."""
|
||||
if self._capacity_only:
|
||||
return
|
||||
if not kv_caches:
|
||||
logger.warning("No KV caches to offload.")
|
||||
return
|
||||
@@ -1541,6 +1560,9 @@ class MooncakeStoreWorker:
|
||||
compute is launched on the compute stream) for better
|
||||
compute-I/O overlap.
|
||||
"""
|
||||
if self._capacity_only:
|
||||
return set(), set()
|
||||
|
||||
# Issue async loads
|
||||
for request in meta.requests:
|
||||
load_spec = request.load_spec
|
||||
@@ -1659,6 +1681,9 @@ class MooncakeStoreWorker:
|
||||
hit covering all ``num_tokens`` is re-derived below the request end so
|
||||
the last token is recomputed for sampling.
|
||||
"""
|
||||
if self._capacity_only:
|
||||
return 0
|
||||
|
||||
token_len = self.coord.align_lookup_length(num_tokens)
|
||||
if not block_hashes or token_len <= 0:
|
||||
return 0
|
||||
|
||||
Reference in New Issue
Block a user