(feat): optionally disable lookup on PD decode (#50498)

Signed-off-by: Jeff Ma <jeffjma@umich.edu>
Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
Jeff (Junze) Ma
2026-08-01 09:41:43 +00:00
committed by GitHub
co-authored by Yifan Qiao
parent 81a42d37b1
commit 3986b967b2
5 changed files with 67 additions and 3 deletions
@@ -19,6 +19,7 @@ def _make_bare_scheduler(
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.lookup_async = False
scheduler.enable_lookup = True
scheduler._block_size = 16
scheduler._hash_block_size = hash_block_size
scheduler.enable_partial_hash_hits = enable_partial_hash_hits
@@ -636,6 +637,30 @@ def test_sub_block_prompt_not_looked_up_without_fine_grained():
assert "req-0" not in scheduler.load_specs
def test_disabled_lookup_reports_no_hit_without_querying_client():
# With enable_lookup=False the connector reports no external hit without
# consulting the lookup client, so admission is never deferred on a store
# lookup. Used by instances that only contribute store capacity.
scheduler = _make_bare_scheduler()
scheduler.enable_lookup = False
scheduler.client = _StubLookupClient(hit_tokens=32)
request = SimpleNamespace(
request_id="req-0",
num_tokens=48,
block_hashes=[b"h0", b"h1", b"h2"],
)
need_to_allocate, load_async = scheduler.get_num_new_matched_tokens(
request, num_computed_tokens=0
)
assert need_to_allocate == 0
assert load_async is False
assert scheduler.client.num_tokens == []
assert scheduler.load_specs == {}
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
@@ -1746,6 +1746,7 @@ def _make_bare_worker(
worker.store = MagicMock()
worker.store.register_buffer.return_value = 0
worker.kv_role = kv_role
worker._capacity_only = False
worker.block_size = block_size
worker.tp_rank = 0
worker.enable_kv_events = False