[KV Connector] Add decode offloading to Mooncake Store consumers (#52466)

Signed-off-by: z-zanez <zhouzh93@mail2.sysu.edu.cn>
Signed-off-by: Guanyi Chen <939416532@qq.com>
Co-authored-by: z-zanez <zhouzh93@mail2.sysu.edu.cn>
This commit is contained in:
Guanyi Chen
2026-08-19 17:40:35 -07:00
committed by GitHub
co-authored by z-zanez
parent d4f4d3f40f
commit bf2866f8bf
7 changed files with 518 additions and 72 deletions
@@ -3,6 +3,8 @@
from types import SimpleNamespace
import pytest
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
MooncakeStoreWorkerMetadata,
@@ -16,10 +18,16 @@ from vllm.v1.core.block_pool import BlockPool
def _make_bare_scheduler(
*, hash_block_size: int = 16, enable_partial_hash_hits: bool = False
*,
hash_block_size: int = 16,
enable_partial_hash_hits: bool = False,
kv_role: str = "kv_both",
save_decode_cache: bool = False,
) -> MooncakeStoreScheduler:
scheduler = object.__new__(MooncakeStoreScheduler)
scheduler.kv_role = "kv_both"
scheduler.kv_role = kv_role
scheduler.save_decode_cache = save_decode_cache
scheduler.enable_kv_events = False
scheduler.lookup_async = False
scheduler.enable_lookup = True
scheduler._block_size = 16
@@ -56,6 +64,74 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None):
)
def _make_decode_scheduler_output(
*, num_computed_tokens: int, num_scheduled_tokens: int = 1
) -> SimpleNamespace:
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
# The block that becomes full was allocated on an earlier step.
new_block_ids=[()],
num_computed_tokens=[num_computed_tokens],
resumed_req_ids=set(),
),
num_scheduled_tokens={"req-0": num_scheduled_tokens},
scheduled_spec_decode_tokens={},
)
def _make_new_scheduler_output() -> SimpleNamespace:
request = SimpleNamespace(
req_id="req-0",
num_computed_tokens=0,
prompt_token_ids=list(range(32)),
prefill_token_ids=None,
block_ids=([0, 1],),
block_hashes=[b"h0", b"h1"],
)
return SimpleNamespace(
request=request,
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[request],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={"req-0": 32},
scheduled_spec_decode_tokens={},
)
def test_scheduler_only_tracks_token_ids_for_kv_events():
for enable_kv_events in (False, True):
scheduler = _make_bare_scheduler()
scheduler.enable_kv_events = enable_kv_events
scheduler._unfinished_requests["req-0"] = (
_make_new_scheduler_output().request,
([0, 1],),
)
meta = scheduler.build_connector_meta(_make_new_scheduler_output())
tracker = scheduler._request_trackers["req-0"]
req_meta = meta.requests[0]
if enable_kv_events:
assert tracker.token_ids == list(range(32))
assert req_meta.token_ids == list(range(32))
assert req_meta.token_ids_start == 0
tracker.token_ids.append(99)
assert req_meta.token_ids == list(range(32))
else:
assert tracker.token_ids is None
assert req_meta.token_ids is None
def _make_preemption_scheduler_output():
return SimpleNamespace(
finished_req_ids=set(),
@@ -103,6 +179,29 @@ def _add_unfinished_request(
)
def _setup_decode_request(
*,
kv_role: str = "kv_consumer",
save_decode_cache: bool = False,
token_len: int = 47,
) -> tuple[MooncakeStoreScheduler, RequestTracker]:
scheduler = _make_bare_scheduler(
kv_role=kv_role, save_decode_cache=save_decode_cache
)
token_ids = list(range(token_len + 1))
_add_unfinished_request(
scheduler,
token_ids=token_ids,
block_hashes=[b"h0", b"h1", b"h2"],
prefill_end_tokens=32,
)
tracker = scheduler._request_trackers["req-0"]
tracker.token_len = token_len
tracker.allocated_block_ids = ([0, 1, 2],)
tracker.token_ids = token_ids[:token_len]
return scheduler, tracker
def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts():
# Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids
# yet, so the tracker's token_len does not advance and num_tokens_to_save
@@ -149,6 +248,92 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
assert tracker.num_saved_tokens == 48
@pytest.mark.parametrize("kv_role", ["kv_consumer", "kv_both"])
def test_decode_tracking_is_skipped_by_default(kv_role):
scheduler, tracker = _setup_decode_request(kv_role=kv_role)
meta = scheduler.build_connector_meta(
_make_decode_scheduler_output(num_computed_tokens=47)
)
assert meta.requests == []
assert tracker.token_len == 47
assert tracker.num_saved_tokens == 32
assert tracker.allocated_block_ids == ([0, 1, 2],)
assert tracker.token_ids == list(range(47))
def test_fresh_consumer_first_decode_save_can_backfill_missing_prompt():
scheduler = _make_bare_scheduler(
kv_role="kv_consumer",
save_decode_cache=True,
)
scheduler.enable_kv_events = True
new_output = _make_new_scheduler_output()
request = new_output.request
request.block_ids = ([0, 1, 2],)
request.block_hashes = [b"h0", b"h1", b"h2"]
request.all_token_ids = list(range(48))
request.num_output_placeholders = 0
scheduler._unfinished_requests["req-0"] = (request, request.block_ids)
# The consumer does not save during prefill, so its first decode save must
# still cover the prompt. The worker deduplicates prompt blocks already in
# the Store and fills any that are missing, preserving a complete prefix.
assert scheduler.build_connector_meta(new_output).requests == []
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
# The first decode step checks/saves the block-aligned prompt. The worker's
# Store dedup turns this into a no-op when the producer already saved it.
[prompt_meta] = scheduler.build_connector_meta(
_make_decode_scheduler_output(num_computed_tokens=32)
).requests
assert prompt_meta.token_len_chunk == 32
assert prompt_meta.token_ids_start == 0
assert prompt_meta.token_ids == list(range(32))
for num_computed_tokens in range(33, 48):
meta = scheduler.build_connector_meta(
_make_decode_scheduler_output(
num_computed_tokens=num_computed_tokens,
)
)
if num_computed_tokens < 47:
assert meta.requests == []
[req_meta] = meta.requests
assert req_meta.token_len_chunk == 48
assert req_meta.token_ids_start == 32
assert req_meta.token_ids == list(range(32, 48))
assert tracker.num_saved_tokens == 48
@pytest.mark.parametrize("token_len, saved_tokens", [(46, 32), (47, 48)])
def test_consumer_saves_only_full_decode_blocks(token_len, saved_tokens):
scheduler, tracker = _setup_decode_request(
save_decode_cache=True, token_len=token_len
)
meta = scheduler.build_connector_meta(
_make_decode_scheduler_output(num_computed_tokens=token_len)
)
assert tracker.token_len == token_len + 1
assert tracker.num_saved_tokens == saved_tokens
if saved_tokens == 32:
assert meta.requests == []
else:
[req_meta] = meta.requests
assert req_meta.can_save is True
assert req_meta.token_len_chunk == 48
assert req_meta.block_ids == ([0, 1, 2],)
assert req_meta.token_ids == list(range(32, 48))
assert req_meta.token_ids_start == 32
tracker.token_ids.append(999)
assert req_meta.token_ids == list(range(32, 48))
def test_preemption_resets_tracker():
scheduler = _make_bare_scheduler()
_add_unfinished_request(
@@ -36,7 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import (
MooncakeStoreConnectorStats,
)
from vllm.v1.core.kv_cache_utils import BlockHash
from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash
class _RecordingBlockHashes:
@@ -230,6 +230,7 @@ def _make_vllm_config(
extra_config: dict[str, object] | None = None,
rank: int = 0,
decode_context_parallel_size: int = 1,
kv_role: str = "kv_both",
) -> SimpleNamespace:
return SimpleNamespace(
model_config=_FakeModelConfig(),
@@ -240,7 +241,9 @@ def _make_vllm_config(
decode_context_parallel_size=decode_context_parallel_size,
prefill_context_parallel_size=1,
),
kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config),
kv_transfer_config=_FakeKVTransferConfig(
kv_role=kv_role, extra_config=extra_config
),
cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10),
kv_events_config=SimpleNamespace(enable_kv_cache_events=False),
speculative_config=None,
@@ -943,15 +946,20 @@ def test_stale_store_job_cannot_touch_a_reused_request_id():
live = _make_store_req("req-a", [b"a0", b"a1"])
live.store_job_id = 2
thread.add_request(live)
thread._retry_token_ids["req-a"] = (32, list(range(32, 64)))
thread.finish_store_job(stale)
thread._record_saved(stale, 64)
thread._mark_request_skipped_for_pressure(stale)
thread._update_retry_token_ids(stale, False, 0, list(range(32)))
thread._update_retry_token_ids(stale, True, 0, None)
assert thread.is_live_store_job(live)
assert not thread.is_live_store_job(stale)
assert thread._saved_offset.get("req-a") is None
assert "req-a" not in thread._skip_store_requests
assert thread._get_retry_token_ids(stale) is None
assert thread._get_retry_token_ids(live) == (32, list(range(32, 64)))
def test_store_recving_thread_reports_failed_block_ids():
@@ -1655,6 +1663,39 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled(
assert w.disk_offload_buffer_budget_bytes is None
def test_save_decode_cache_keeps_transfer_path_enabled(tmp_path, monkeypatch):
store = MagicMock()
store.setup.return_value = 0
_install_fake_mooncake(monkeypatch, store)
_patch_worker_runtime(monkeypatch)
monkeypatch.setenv(
"MOONCAKE_CONFIG_PATH",
_write_mooncake_config(
tmp_path,
{
"metadata_server": "http://metadata/endpoint",
"protocol": "tcp",
"device_name": "",
"master_server_address": "10.0.0.7:50051",
},
),
)
w = worker.MooncakeStoreWorker(
_make_vllm_config(
kv_role="kv_consumer",
extra_config={
"enable_lookup": False,
"save_decode_cache": True,
},
),
_make_kv_cache_config(),
)
assert w.can_put is True
assert w._capacity_only is False
def test_requester_worker_init_builds_replicate_config_for_preferred_segment(
tmp_path,
monkeypatch,
@@ -1962,7 +2003,7 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
store = MagicMock()
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.side_effect = (
lambda keys, addrs, sizes, *_args: [256] * len(keys)
lambda keys, addrs, sizes, *_args: ([256] * len(keys))
)
full_spec = FullAttentionSpec(
@@ -2176,6 +2217,85 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
assert swa_event.block_hashes == [maybe_convert_block_hash(BlockHash(hs[3]))]
def _make_event_store_req(token_len: int, token_ids_start: int = 0) -> ReqMeta:
num_blocks = token_len // 16
return ReqMeta(
req_id="r0",
token_len_chunk=token_len,
block_ids=(list(range(num_blocks)),),
block_hashes=[f"a{i}".encode() for i in range(num_blocks)],
can_save=True,
token_ids=list(range(token_ids_start, token_len)),
token_ids_start=token_ids_start,
)
@pytest.mark.parametrize(
("saved_offset", "put_step", "exists", "stored_indices", "parent_indices"),
[
pytest.param(16, 1, [0, 0, 0], [1, 2, 3], [0, 1, 2], id="suffix"),
pytest.param(0, 1, [1, 0, 1, 0], [1, 3], [0, 2], id="dedup-holes"),
pytest.param(0, 2, [0, 0], [0, 2], [None, 1], id="tp-stride"),
],
)
def test_store_sending_thread_kv_events_use_request_chain_parents(
saved_offset, put_step, exists, stored_indices, parent_indices
):
store = MagicMock()
store.batch_is_exist.return_value = exists
store.batch_put_from_multi_buffers.return_value = [256] * len(stored_indices)
thread = _make_store_sending_thread(store, put_step=put_step)
thread.enable_kv_event = True
thread._saved_offset["r0"] = saved_offset
_run_store_req(thread, _make_event_store_req(64, saved_offset))
events = thread.get_kv_events()
assert [event.block_hashes for event in events] == [
[maybe_convert_block_hash(BlockHash(f"a{i}".encode()))] for i in stored_indices
]
assert [event.parent_block_hash for event in events] == [
(None if i is None else maybe_convert_block_hash(BlockHash(f"a{i}".encode())))
for i in parent_indices
]
assert thread._retry_token_ids == {}
def test_store_sending_thread_kv_events_retry_without_covered_tokens():
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
thread.enable_kv_event = True
_run_store_req(thread, _make_event_store_req(32, 16))
retry_event, suffix_event = thread.get_kv_events()
assert retry_event.token_ids == []
assert suffix_event.token_ids == list(range(16, 32))
def test_store_sending_thread_kv_events_recover_suffix_after_put_failure():
store = MagicMock()
store.batch_is_exist.side_effect = ([0], [0, 0])
store.batch_put_from_multi_buffers.side_effect = ([-1], [256, 256])
thread = _make_store_sending_thread(store)
thread.enable_kv_event = True
_run_store_req(thread, _make_event_store_req(16))
assert thread.get_kv_events() == []
assert thread._retry_token_ids["r0"] == (0, list(range(16)))
_run_store_req(thread, _make_event_store_req(32, 16))
retry_event, suffix_event = thread.get_kv_events()
assert retry_event.token_ids == list(range(16))
assert suffix_event.token_ids == list(range(16, 32))
assert thread._retry_token_ids == {}
assert thread._saved_offset["r0"] == 32
def _auto_set_ready_event(*args, **kwargs):
"""Side effect for mocked thread constructors that auto-sets ready_event."""
for arg in args:
@@ -2214,6 +2334,7 @@ def _make_bare_worker(
num_gpu_blocks: int = 10,
block_size: int = 16,
kv_role: str = "kv_both",
save_decode_cache: bool = False,
) -> mooncake_store_worker.MooncakeStoreWorker:
"""Construct a MooncakeStoreWorker via __new__, bypassing __init__.
@@ -2227,14 +2348,17 @@ def _make_bare_worker(
worker.store = MagicMock()
worker.store.register_buffer.return_value = 0
worker.kv_role = kv_role
worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache
worker._capacity_only = False
worker.block_size = block_size
worker.tp_rank = 0
worker.enable_kv_events = False
worker.load_async = True
worker.kv_send_thread = None
worker.kv_recv_threads = []
worker.num_recv_threads = 1
worker.recv_request_queue = queue.Queue()
worker.finished_store_req = set()
worker.tp_size = 1
worker.num_kv_head = 1
worker.pp_size = 1
@@ -2743,6 +2867,36 @@ def test_lookup_applies_swa_mask_before_accessing_hashes():
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("save_decode_cache", [False, True])
def test_consumer_starts_send_thread_only_when_put_is_enabled(save_decode_cache):
num_blocks = 10
tensor = torch.zeros(num_blocks, 64, dtype=torch.float16)
worker = _make_bare_worker(
kv_role="kv_consumer", save_decode_cache=save_decode_cache
)
_register_with_mocked_threads(worker, {"layer0": tensor})
assert (worker.kv_send_thread is not None) == save_decode_cache
def test_putting_consumer_queues_decode_save():
w = _make_bare_worker(kv_role="kv_consumer", save_decode_cache=True)
send_thread = MagicMock()
w.kv_send_thread = send_thread
req = _make_store_req("decode-req", [b"h0", b"h1"])
req.store_job_id = 1
meta = mooncake_store_worker.MooncakeStoreConnectorMetadata(set(), set())
meta.add_request(req)
with patch.object(torch.cuda, "Event") as event_cls:
event = event_cls.return_value
w.get_finished(set(), meta)
event.record.assert_called_once_with()
send_thread.add_request.assert_called_once_with(req)
assert req.current_event is event
def test_register_kv_caches_blocks_first_single_segment():
"""Blocks-first layout (FlashInfer/MLA): one segment per layer."""
num_blocks = 10