diff --git a/docs/features/mooncake_store_connector_usage.md b/docs/features/mooncake_store_connector_usage.md index a07716dea6..800b814250 100644 --- a/docs/features/mooncake_store_connector_usage.md +++ b/docs/features/mooncake_store_connector_usage.md @@ -136,6 +136,25 @@ vllm serve meta-llama/Llama-3.1-8B-Instruct \ }' ``` +To also offload newly completed decode KV blocks, add the following extra +configuration to the decoder's `MooncakeStoreConnector` entry. This currently +requires the prefill and decode instances to use the same tensor-parallel size +and compatible KV-cache topology. + +When decode processing starts, the consumer checks the block-aligned prompt +prefix and fills any blocks missing from the Store. Subsequent saves append +newly completed decode blocks. This keeps a complete, reusable prefix in the +Store and also covers deployments where prompt KV is delivered directly by +`MooncakeConnector` instead of through the Store. + +```json +{ + "kv_connector_extra_config": { + "save_decode_cache": true + } +} +``` + **Proxy:** A disaggregation proxy is required to route requests between prefiller and decoder nodes. The proxy assigns `do_remote_prefill=True` / `do_remote_decode=True` to coordinate P2P transfer via `MooncakeConnector`. Refer to the [MooncakeConnector usage guide](mooncake_connector_usage.md) for proxy setup details. @@ -230,6 +249,10 @@ Strict isolation requires a Mooncake master started with `--enable_multi_tenants - `enable_cross_layers_blocks` (bool): Enable cross-layer block packing for reduced store operations. Default: `false`. - `lookup_rpc_port` (int): Custom port for the ZMQ lookup RPC socket. Default: `0`. - `cache_prefix` (str): Namespace prepended to every store key. Lets separate deployments share one Mooncake master without polluting each other — instances configured with different prefixes never see each other's cached blocks, even for identical prompts. All instances that should share a prefix cache must use the same value. Default: `""` (no prefix; keys are byte-identical to the unprefixed format). +- `save_decode_cache` (bool): Enable offloading decode tokens' KV cache. A `kv_consumer` does not save during prefill; when decode starts, it fills any missing block-aligned prompt prefix before appending completed decode blocks. Default: `false`. + +Decode offloading uses the existing TP-rank key namespace. Cross-TP sharing is +not yet supported. ## Notes diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 998a8408e6..1f02d18900 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -3,6 +3,8 @@ from types import SimpleNamespace +import pytest + from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( LoadSpec, MooncakeStoreWorkerMetadata, @@ -16,10 +18,16 @@ from vllm.v1.core.block_pool import BlockPool def _make_bare_scheduler( - *, hash_block_size: int = 16, enable_partial_hash_hits: bool = False + *, + hash_block_size: int = 16, + enable_partial_hash_hits: bool = False, + kv_role: str = "kv_both", + save_decode_cache: bool = False, ) -> MooncakeStoreScheduler: scheduler = object.__new__(MooncakeStoreScheduler) - scheduler.kv_role = "kv_both" + scheduler.kv_role = kv_role + scheduler.save_decode_cache = save_decode_cache + scheduler.enable_kv_events = False scheduler.lookup_async = False scheduler.enable_lookup = True scheduler._block_size = 16 @@ -56,6 +64,74 @@ def _make_scheduler_output(*, scheduled_spec_tokens: list[int] | None): ) +def _make_decode_scheduler_output( + *, num_computed_tokens: int, num_scheduled_tokens: int = 1 +) -> SimpleNamespace: + return SimpleNamespace( + finished_req_ids=set(), + preempted_req_ids=set(), + scheduled_new_reqs=[], + scheduled_cached_reqs=SimpleNamespace( + req_ids=["req-0"], + # The block that becomes full was allocated on an earlier step. + new_block_ids=[()], + num_computed_tokens=[num_computed_tokens], + resumed_req_ids=set(), + ), + num_scheduled_tokens={"req-0": num_scheduled_tokens}, + scheduled_spec_decode_tokens={}, + ) + + +def _make_new_scheduler_output() -> SimpleNamespace: + request = SimpleNamespace( + req_id="req-0", + num_computed_tokens=0, + prompt_token_ids=list(range(32)), + prefill_token_ids=None, + block_ids=([0, 1],), + block_hashes=[b"h0", b"h1"], + ) + return SimpleNamespace( + request=request, + finished_req_ids=set(), + preempted_req_ids=set(), + scheduled_new_reqs=[request], + scheduled_cached_reqs=SimpleNamespace( + req_ids=[], + new_block_ids=[], + num_computed_tokens=[], + resumed_req_ids=set(), + ), + num_scheduled_tokens={"req-0": 32}, + scheduled_spec_decode_tokens={}, + ) + + +def test_scheduler_only_tracks_token_ids_for_kv_events(): + for enable_kv_events in (False, True): + scheduler = _make_bare_scheduler() + scheduler.enable_kv_events = enable_kv_events + scheduler._unfinished_requests["req-0"] = ( + _make_new_scheduler_output().request, + ([0, 1],), + ) + + meta = scheduler.build_connector_meta(_make_new_scheduler_output()) + + tracker = scheduler._request_trackers["req-0"] + req_meta = meta.requests[0] + if enable_kv_events: + assert tracker.token_ids == list(range(32)) + assert req_meta.token_ids == list(range(32)) + assert req_meta.token_ids_start == 0 + tracker.token_ids.append(99) + assert req_meta.token_ids == list(range(32)) + else: + assert tracker.token_ids is None + assert req_meta.token_ids is None + + def _make_preemption_scheduler_output(): return SimpleNamespace( finished_req_ids=set(), @@ -103,6 +179,29 @@ def _add_unfinished_request( ) +def _setup_decode_request( + *, + kv_role: str = "kv_consumer", + save_decode_cache: bool = False, + token_len: int = 47, +) -> tuple[MooncakeStoreScheduler, RequestTracker]: + scheduler = _make_bare_scheduler( + kv_role=kv_role, save_decode_cache=save_decode_cache + ) + token_ids = list(range(token_len + 1)) + _add_unfinished_request( + scheduler, + token_ids=token_ids, + block_hashes=[b"h0", b"h1", b"h2"], + prefill_end_tokens=32, + ) + tracker = scheduler._request_trackers["req-0"] + tracker.token_len = token_len + tracker.allocated_block_ids = ([0, 1, 2],) + tracker.token_ids = token_ids[:token_len] + return scheduler, tracker + + def test_cached_request_with_spec_decode_does_not_save_scheduled_drafts(): # Drafts in scheduled_spec_decode_tokens are not appended to all_token_ids # yet, so the tracker's token_len does not advance and num_tokens_to_save @@ -149,6 +248,92 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap(): assert tracker.num_saved_tokens == 48 +@pytest.mark.parametrize("kv_role", ["kv_consumer", "kv_both"]) +def test_decode_tracking_is_skipped_by_default(kv_role): + scheduler, tracker = _setup_decode_request(kv_role=kv_role) + + meta = scheduler.build_connector_meta( + _make_decode_scheduler_output(num_computed_tokens=47) + ) + + assert meta.requests == [] + assert tracker.token_len == 47 + assert tracker.num_saved_tokens == 32 + assert tracker.allocated_block_ids == ([0, 1, 2],) + assert tracker.token_ids == list(range(47)) + + +def test_fresh_consumer_first_decode_save_can_backfill_missing_prompt(): + scheduler = _make_bare_scheduler( + kv_role="kv_consumer", + save_decode_cache=True, + ) + scheduler.enable_kv_events = True + new_output = _make_new_scheduler_output() + request = new_output.request + request.block_ids = ([0, 1, 2],) + request.block_hashes = [b"h0", b"h1", b"h2"] + request.all_token_ids = list(range(48)) + request.num_output_placeholders = 0 + scheduler._unfinished_requests["req-0"] = (request, request.block_ids) + + # The consumer does not save during prefill, so its first decode save must + # still cover the prompt. The worker deduplicates prompt blocks already in + # the Store and fills any that are missing, preserving a complete prefix. + assert scheduler.build_connector_meta(new_output).requests == [] + tracker = scheduler._request_trackers["req-0"] + assert tracker.num_saved_tokens == 0 + + # The first decode step checks/saves the block-aligned prompt. The worker's + # Store dedup turns this into a no-op when the producer already saved it. + [prompt_meta] = scheduler.build_connector_meta( + _make_decode_scheduler_output(num_computed_tokens=32) + ).requests + assert prompt_meta.token_len_chunk == 32 + assert prompt_meta.token_ids_start == 0 + assert prompt_meta.token_ids == list(range(32)) + + for num_computed_tokens in range(33, 48): + meta = scheduler.build_connector_meta( + _make_decode_scheduler_output( + num_computed_tokens=num_computed_tokens, + ) + ) + if num_computed_tokens < 47: + assert meta.requests == [] + + [req_meta] = meta.requests + assert req_meta.token_len_chunk == 48 + assert req_meta.token_ids_start == 32 + assert req_meta.token_ids == list(range(32, 48)) + assert tracker.num_saved_tokens == 48 + + +@pytest.mark.parametrize("token_len, saved_tokens", [(46, 32), (47, 48)]) +def test_consumer_saves_only_full_decode_blocks(token_len, saved_tokens): + scheduler, tracker = _setup_decode_request( + save_decode_cache=True, token_len=token_len + ) + + meta = scheduler.build_connector_meta( + _make_decode_scheduler_output(num_computed_tokens=token_len) + ) + + assert tracker.token_len == token_len + 1 + assert tracker.num_saved_tokens == saved_tokens + if saved_tokens == 32: + assert meta.requests == [] + else: + [req_meta] = meta.requests + assert req_meta.can_save is True + assert req_meta.token_len_chunk == 48 + assert req_meta.block_ids == ([0, 1, 2],) + assert req_meta.token_ids == list(range(32, 48)) + assert req_meta.token_ids_start == 32 + tracker.token_ids.append(999) + assert req_meta.token_ids == list(range(32, 48)) + + def test_preemption_resets_tracker(): scheduler = _make_bare_scheduler() _add_unfinished_request( diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 50d2ee1614..39c0f8eb4a 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -36,7 +36,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.metrics import ( MooncakeStoreConnectorStats, ) -from vllm.v1.core.kv_cache_utils import BlockHash +from vllm.v1.core.kv_cache_utils import BlockHash, maybe_convert_block_hash class _RecordingBlockHashes: @@ -230,6 +230,7 @@ def _make_vllm_config( extra_config: dict[str, object] | None = None, rank: int = 0, decode_context_parallel_size: int = 1, + kv_role: str = "kv_both", ) -> SimpleNamespace: return SimpleNamespace( model_config=_FakeModelConfig(), @@ -240,7 +241,9 @@ def _make_vllm_config( decode_context_parallel_size=decode_context_parallel_size, prefill_context_parallel_size=1, ), - kv_transfer_config=_FakeKVTransferConfig(extra_config=extra_config), + kv_transfer_config=_FakeKVTransferConfig( + kv_role=kv_role, extra_config=extra_config + ), cache_config=SimpleNamespace(block_size=16, num_gpu_blocks=10), kv_events_config=SimpleNamespace(enable_kv_cache_events=False), speculative_config=None, @@ -943,15 +946,20 @@ def test_stale_store_job_cannot_touch_a_reused_request_id(): live = _make_store_req("req-a", [b"a0", b"a1"]) live.store_job_id = 2 thread.add_request(live) + thread._retry_token_ids["req-a"] = (32, list(range(32, 64))) thread.finish_store_job(stale) thread._record_saved(stale, 64) thread._mark_request_skipped_for_pressure(stale) + thread._update_retry_token_ids(stale, False, 0, list(range(32))) + thread._update_retry_token_ids(stale, True, 0, None) assert thread.is_live_store_job(live) assert not thread.is_live_store_job(stale) assert thread._saved_offset.get("req-a") is None assert "req-a" not in thread._skip_store_requests + assert thread._get_retry_token_ids(stale) is None + assert thread._get_retry_token_ids(live) == (32, list(range(32, 64))) def test_store_recving_thread_reports_failed_block_ids(): @@ -1655,6 +1663,39 @@ def test_requester_worker_init_skips_disk_budget_when_offload_disabled( assert w.disk_offload_buffer_budget_bytes is None +def test_save_decode_cache_keeps_transfer_path_enabled(tmp_path, monkeypatch): + store = MagicMock() + store.setup.return_value = 0 + _install_fake_mooncake(monkeypatch, store) + _patch_worker_runtime(monkeypatch) + monkeypatch.setenv( + "MOONCAKE_CONFIG_PATH", + _write_mooncake_config( + tmp_path, + { + "metadata_server": "http://metadata/endpoint", + "protocol": "tcp", + "device_name": "", + "master_server_address": "10.0.0.7:50051", + }, + ), + ) + + w = worker.MooncakeStoreWorker( + _make_vllm_config( + kv_role="kv_consumer", + extra_config={ + "enable_lookup": False, + "save_decode_cache": True, + }, + ), + _make_kv_cache_config(), + ) + + assert w.can_put is True + assert w._capacity_only is False + + def test_requester_worker_init_builds_replicate_config_for_preferred_segment( tmp_path, monkeypatch, @@ -1962,7 +2003,7 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window(): store = MagicMock() store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) store.batch_put_from_multi_buffers.side_effect = ( - lambda keys, addrs, sizes, *_args: [256] * len(keys) + lambda keys, addrs, sizes, *_args: ([256] * len(keys)) ) full_spec = FullAttentionSpec( @@ -2176,6 +2217,85 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata(): assert swa_event.block_hashes == [maybe_convert_block_hash(BlockHash(hs[3]))] +def _make_event_store_req(token_len: int, token_ids_start: int = 0) -> ReqMeta: + num_blocks = token_len // 16 + return ReqMeta( + req_id="r0", + token_len_chunk=token_len, + block_ids=(list(range(num_blocks)),), + block_hashes=[f"a{i}".encode() for i in range(num_blocks)], + can_save=True, + token_ids=list(range(token_ids_start, token_len)), + token_ids_start=token_ids_start, + ) + + +@pytest.mark.parametrize( + ("saved_offset", "put_step", "exists", "stored_indices", "parent_indices"), + [ + pytest.param(16, 1, [0, 0, 0], [1, 2, 3], [0, 1, 2], id="suffix"), + pytest.param(0, 1, [1, 0, 1, 0], [1, 3], [0, 2], id="dedup-holes"), + pytest.param(0, 2, [0, 0], [0, 2], [None, 1], id="tp-stride"), + ], +) +def test_store_sending_thread_kv_events_use_request_chain_parents( + saved_offset, put_step, exists, stored_indices, parent_indices +): + store = MagicMock() + store.batch_is_exist.return_value = exists + store.batch_put_from_multi_buffers.return_value = [256] * len(stored_indices) + thread = _make_store_sending_thread(store, put_step=put_step) + thread.enable_kv_event = True + + thread._saved_offset["r0"] = saved_offset + _run_store_req(thread, _make_event_store_req(64, saved_offset)) + + events = thread.get_kv_events() + assert [event.block_hashes for event in events] == [ + [maybe_convert_block_hash(BlockHash(f"a{i}".encode()))] for i in stored_indices + ] + assert [event.parent_block_hash for event in events] == [ + (None if i is None else maybe_convert_block_hash(BlockHash(f"a{i}".encode()))) + for i in parent_indices + ] + assert thread._retry_token_ids == {} + + +def test_store_sending_thread_kv_events_retry_without_covered_tokens(): + store = MagicMock() + store.batch_is_exist.return_value = [0, 0] + store.batch_put_from_multi_buffers.return_value = [256, 256] + thread = _make_store_sending_thread(store) + thread.enable_kv_event = True + + _run_store_req(thread, _make_event_store_req(32, 16)) + + retry_event, suffix_event = thread.get_kv_events() + assert retry_event.token_ids == [] + assert suffix_event.token_ids == list(range(16, 32)) + + +def test_store_sending_thread_kv_events_recover_suffix_after_put_failure(): + store = MagicMock() + store.batch_is_exist.side_effect = ([0], [0, 0]) + store.batch_put_from_multi_buffers.side_effect = ([-1], [256, 256]) + thread = _make_store_sending_thread(store) + thread.enable_kv_event = True + + _run_store_req(thread, _make_event_store_req(16)) + + assert thread.get_kv_events() == [] + assert thread._retry_token_ids["r0"] == (0, list(range(16))) + + _run_store_req(thread, _make_event_store_req(32, 16)) + + retry_event, suffix_event = thread.get_kv_events() + assert retry_event.token_ids == list(range(16)) + assert suffix_event.token_ids == list(range(16, 32)) + assert thread._retry_token_ids == {} + assert thread._saved_offset["r0"] == 32 + + def _auto_set_ready_event(*args, **kwargs): """Side effect for mocked thread constructors that auto-sets ready_event.""" for arg in args: @@ -2214,6 +2334,7 @@ def _make_bare_worker( num_gpu_blocks: int = 10, block_size: int = 16, kv_role: str = "kv_both", + save_decode_cache: bool = False, ) -> mooncake_store_worker.MooncakeStoreWorker: """Construct a MooncakeStoreWorker via __new__, bypassing __init__. @@ -2227,14 +2348,17 @@ def _make_bare_worker( worker.store = MagicMock() worker.store.register_buffer.return_value = 0 worker.kv_role = kv_role + worker.can_put = kv_role in ("kv_producer", "kv_both") or save_decode_cache worker._capacity_only = False worker.block_size = block_size worker.tp_rank = 0 worker.enable_kv_events = False + worker.load_async = True worker.kv_send_thread = None worker.kv_recv_threads = [] worker.num_recv_threads = 1 worker.recv_request_queue = queue.Queue() + worker.finished_store_req = set() worker.tp_size = 1 worker.num_kv_head = 1 worker.pp_size = 1 @@ -2743,6 +2867,36 @@ def test_lookup_applies_swa_mask_before_accessing_hashes(): # --------------------------------------------------------------------------- +@pytest.mark.parametrize("save_decode_cache", [False, True]) +def test_consumer_starts_send_thread_only_when_put_is_enabled(save_decode_cache): + num_blocks = 10 + tensor = torch.zeros(num_blocks, 64, dtype=torch.float16) + + worker = _make_bare_worker( + kv_role="kv_consumer", save_decode_cache=save_decode_cache + ) + _register_with_mocked_threads(worker, {"layer0": tensor}) + assert (worker.kv_send_thread is not None) == save_decode_cache + + +def test_putting_consumer_queues_decode_save(): + w = _make_bare_worker(kv_role="kv_consumer", save_decode_cache=True) + send_thread = MagicMock() + w.kv_send_thread = send_thread + req = _make_store_req("decode-req", [b"h0", b"h1"]) + req.store_job_id = 1 + meta = mooncake_store_worker.MooncakeStoreConnectorMetadata(set(), set()) + meta.add_request(req) + + with patch.object(torch.cuda, "Event") as event_cls: + event = event_cls.return_value + w.get_finished(set(), meta) + + event.record.assert_called_once_with() + send_thread.add_request.assert_called_once_with(req) + assert req.current_event is event + + def test_register_kv_caches_blocks_first_single_segment(): """Blocks-first layout (FlashInfer/MLA): one segment per layer.""" num_blocks = 10 diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index 297fc22ad8..e35eb6adb3 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -141,12 +141,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): assert vllm_config.kv_transfer_config is not None assert kv_cache_config is not None, "kv_cache_config is required" self.kv_role = vllm_config.kv_transfer_config.kv_role + extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config + save_decode_cache = extra_config.get("save_decode_cache", False) # Capacity-only: contributes its segment to the store pool but transfers # no KV, so the KV-cache-shape invariants below cannot be reached. - self._capacity_only = self.kv_role == "kv_consumer" and not ( - vllm_config.kv_transfer_config.kv_connector_extra_config.get( - "enable_lookup", True - ) + self._capacity_only = ( + self.kv_role == "kv_consumer" + and not extra_config.get("enable_lookup", True) + and not save_decode_cache ) if not self._capacity_only: self._validate_kv_cache_config(vllm_config, kv_cache_config) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index 55ffc040cf..725da61e79 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -362,10 +362,11 @@ class ReqMeta: can_save: bool | None = None load_spec: LoadSpec | None = None - is_last_chunk: bool | None = None current_event: torch.cuda.Event | None = None token_ids: list[int] | None = None + # Absolute request offset represented by token_ids[0]. + token_ids_start: int = 0 num_prompt_tokens: int | None = None # Identifies this store job for the engine's lifetime. A request id cannot # serve that purpose: it is reused once a preempted request resumes, so it @@ -384,14 +385,14 @@ class ReqMeta: load_spec: LoadSpec | None = None, skip_save: bool | None = False, block_hashes: list[BlockHash] | None = None, - is_last_chunk: bool | None = None, ) -> "ReqMeta | None": """Create ReqMeta from a RequestTracker.""" if block_hashes is None: block_hashes = [] input_token_len = tracker.token_len - chunk_boundary = cdiv(tracker.num_saved_tokens + 1, block_size) * block_size + token_ids_start = tracker.num_saved_tokens + chunk_boundary = cdiv(token_ids_start + 1, block_size) * block_size num_tokens_to_save = input_token_len // block_size * block_size skip_save = skip_save or num_tokens_to_save < chunk_boundary @@ -408,8 +409,10 @@ class ReqMeta: tracker.num_saved_tokens = num_tokens_to_save token_ids = None - if tracker.token_ids: - token_ids = tracker.token_ids + if tracker.token_ids and not skip_save: + # Scheduler tracking continues while this job is handled by an + # asynchronous worker, so metadata must own a stable snapshot. + token_ids = tracker.token_ids[token_ids_start:num_tokens_to_save] if load_spec is not None and load_spec.can_load: logger.debug( @@ -433,8 +436,8 @@ class ReqMeta: can_save=not skip_save, load_spec=load_spec, block_hashes=block_hashes, - is_last_chunk=is_last_chunk, token_ids=token_ids, + token_ids_start=token_ids_start, num_prompt_tokens=tracker.prefill_end_tokens, ) diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index f583b3fcfd..c26e25fc5a 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -63,6 +63,11 @@ class MooncakeStoreScheduler: self.lookup_async = kvc_extra_config.get("lookup_async", False) # Skips lookup CPU cost on instances that never load KV from the store. self.enable_lookup = kvc_extra_config.get("enable_lookup", True) + self.save_decode_cache = kvc_extra_config.get("save_decode_cache", False) + kv_event_config = vllm_config.kv_events_config + self.enable_kv_events = bool( + kv_event_config and kv_event_config.enable_kv_cache_events + ) self.client = LookupKeyClient(vllm_config) # Align with the engine's own scheduler_block_size and hash_block_size. @@ -182,7 +187,8 @@ class MooncakeStoreScheduler: self, scheduler_output: SchedulerOutput ) -> KVConnectorMetadata: """Build connector metadata for this scheduler step.""" - force_skip_save = self.kv_role == "kv_consumer" + is_consumer = self.kv_role == "kv_consumer" + can_process_cached = not is_consumer or self.save_decode_cache for finished_req_id in scheduler_output.finished_req_ids: self.client.discard(finished_req_id) @@ -227,37 +233,38 @@ class MooncakeStoreScheduler: token_len=num_tokens_to_compute, allocated_block_ids=unfolded_block_ids, num_saved_tokens=0, - token_ids=prefill_tokens[:num_tokens_to_compute], + token_ids=( + prefill_tokens[:num_tokens_to_compute] + if self.enable_kv_events + else None + ), prefill_end_tokens=len(prefill_tokens), ) self._request_trackers[request.req_id] = request_tracker - last_chunk_tokens_num = ( - len(prefill_tokens) // self._block_size * self._block_size - ) - req_meta = ReqMeta.from_request_tracker( request_tracker, self._block_size, load_spec=load_spec, - skip_save=force_skip_save, + # A consumer may write decode KV without becoming a prefill + # producer. Loads are still carried by the same metadata. + skip_save=is_consumer, block_hashes=request_real.block_hashes, - is_last_chunk=(request_tracker.token_len >= last_chunk_tokens_num), ) if req_meta is not None: meta.add_request(req_meta) # Handle cached (running, or MRV1 resumed-from-preemption) requests cached_reqs = scheduler_output.scheduled_cached_reqs - if not force_skip_save: + if can_process_cached: for i, req_id in enumerate(cached_reqs.req_ids): new_block_ids = cached_reqs.new_block_ids[i] - if not new_block_ids: - continue req_meta = None if req_id in cached_reqs.resumed_req_ids: # Resumed after preemption + if not new_block_ids: + continue if isinstance(new_block_ids, tuple): new_block_ids = tuple(b.copy() for b in new_block_ids) else: @@ -277,27 +284,33 @@ class MooncakeStoreScheduler: token_len=num_tokens_to_compute, allocated_block_ids=new_block_ids, num_saved_tokens=0, - token_ids=prefill_tokens[:num_tokens_to_compute].copy(), + token_ids=( + prefill_tokens[:num_tokens_to_compute] + if self.enable_kv_events + else None + ), prefill_end_tokens=len(prefill_tokens), ) self._request_trackers[req_id] = request_tracker - last_chunk_tokens_num = ( - len(prefill_tokens) // self._block_size * self._block_size - ) req_meta = ReqMeta.from_request_tracker( request_tracker, self._block_size, load_spec=load_spec, - skip_save=force_skip_save, + skip_save=is_consumer, block_hashes=request_real.block_hashes, - is_last_chunk=( - request_tracker.token_len >= last_chunk_tokens_num - ), ) else: # Decode/chunked request request_tracker = self._request_trackers[req_id] + num_computed_token = cached_reqs.num_computed_tokens[i] + # Use the tracker's snapshot of the prefill range so resumed + # requests keep saving past the original prompt boundary. + prefill_end = request_tracker.prefill_end_tokens + is_decode = num_computed_token >= prefill_end + if is_decode and not self.save_decode_cache: + continue + num_new_tokens = scheduler_output.num_scheduled_tokens[req_id] req_tuple = self._unfinished_requests.get(req_id) if req_tuple: @@ -307,30 +320,26 @@ class MooncakeStoreScheduler: num_current_tokens : num_current_tokens + num_new_tokens ] request_tracker.token_len += len(new_token_ids) + if request_tracker.token_ids is not None: + request_tracker.token_ids.extend(new_token_ids) else: raise ValueError( f"Request {req_id} is not in _unfinished_requests" ) - num_computed_token = cached_reqs.num_computed_tokens[i] - # Use the tracker's snapshot of the prefill range so resumed - # requests keep saving past the original prompt boundary. - prefill_end = request_tracker.prefill_end_tokens - if num_computed_token >= prefill_end: + # A block is usually allocated before the step that fills + # it, so reaching a save boundary does not imply that this + # step has new block ids. + if new_block_ids: + request_tracker.update(new_block_ids) + if is_consumer and not is_decode: continue - request_tracker.update(new_block_ids) - last_chunk_tokens_num = ( - prefill_end // self._block_size * self._block_size - ) req_meta = ReqMeta.from_request_tracker( request_tracker, self._block_size, load_spec=None, - skip_save=force_skip_save, + skip_save=False, block_hashes=unfinished_req.block_hashes, - is_last_chunk=( - request_tracker.token_len >= last_chunk_tokens_num - ), ) if req_meta is not None: @@ -370,7 +379,7 @@ class MooncakeStoreScheduler: # emit an offload-only ReqMeta (token_len_chunk=0 skips the normal # save; can_save=True takes the normal enqueue path). step_partial_tails = getattr(scheduler_output, "partial_tail_offloads", None) - if step_partial_tails and not force_skip_save: + if step_partial_tails and not is_consumer: pending = dict(step_partial_tails) for req_meta in meta.requests: if req_meta.can_save: diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 83c6d16e1b..efc93f7d94 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -530,6 +530,9 @@ class KVCacheStoreSendingThread(KVTransferThread): # Per-request high-water mark of tokens actually persisted; the next # batch resumes here, so pressure-skipped or failed ranges are retried. self._saved_offset: dict[str, int] = {} + # Retained only after a failed store so retry events can recover the + # token suffix without full snapshots on the normal path. + self._retry_token_ids: dict[str, tuple[int, list[int]]] = {} def add_request(self, request: ReqMeta) -> None: # Register before enqueueing so a job is never picked up unledgered. @@ -552,6 +555,7 @@ class KVCacheStoreSendingThread(KVTransferThread): del self.stored_requests[req_id] self._skip_store_requests.discard(req_id) self._saved_offset.pop(req_id, None) + self._retry_token_ids.pop(req_id, None) def finish_store_job(self, req_meta: ReqMeta) -> None: """Retire a job from the ledger and report its blocks as no longer read. @@ -585,6 +589,36 @@ class KVCacheStoreSendingThread(KVTransferThread): if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()): self._saved_offset[req_meta.req_id] = token_len + def _get_retry_token_ids(self, req_meta: ReqMeta) -> tuple[int, list[int]] | None: + """Return retry state only if this store job is still live.""" + with self.done_task_lock: + if req_meta.store_job_id not in self.stored_requests.get( + req_meta.req_id, () + ): + return None + return self._retry_token_ids.get(req_meta.req_id) + + def _update_retry_token_ids( + self, + req_meta: ReqMeta, + save_completed: bool, + token_ids_start: int, + event_token_ids: list[int] | None, + ) -> None: + """Update retry state without letting a stale job touch a reused ID.""" + with self.done_task_lock: + if req_meta.store_job_id not in self.stored_requests.get( + req_meta.req_id, () + ): + return + if save_completed: + self._retry_token_ids.pop(req_meta.req_id, None) + elif event_token_ids is not None: + self._retry_token_ids[req_meta.req_id] = ( + token_ids_start, + event_token_ids, + ) + def _should_skip_request(self, req_id: str) -> bool: with self.done_task_lock: return self._store_pressure_active and req_id in self._skip_store_requests @@ -785,18 +819,30 @@ class KVCacheStoreSendingThread(KVTransferThread): def _handle_request(self, req_meta: ReqMeta): # The single `finally` is the only way out, so the scheduler releases # this job's GPU block references however the job ends. + save_completed = False + token_len = 0 + req_id = req_meta.req_id + event_token_ids = req_meta.token_ids + token_ids_start = req_meta.token_ids_start try: # Cache hits are always a multiple of ``lcm_block_size`` tokens, # which is also ``store_mask``'s precondition. lcm_block_size = self.coord.lcm_block_size token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size block_ids_per_group = req_meta.block_ids - req_id = req_meta.req_id current_event = req_meta.current_event if not self.is_live_store_job(req_meta): return + if self.enable_kv_event: + retry_token_ids = self._get_retry_token_ids(req_meta) + if retry_token_ids is not None and event_token_ids is not None: + retry_start, retry_ids = retry_token_ids + if retry_start + len(retry_ids) == token_ids_start: + event_token_ids = retry_ids + event_token_ids + token_ids_start = retry_start + if self._should_skip_request(req_id): logger.debug( "Skipping Mooncake store for request %s while CPU/disk " @@ -852,6 +898,7 @@ class KVCacheStoreSendingThread(KVTransferThread): if not keys: self._record_saved(req_meta, token_len) + save_completed = True return # Check which blocks already exist (dedup) @@ -878,6 +925,7 @@ class KVCacheStoreSendingThread(KVTransferThread): if not missing_indices: self._record_saved(req_meta, token_len) + save_completed = True return if len(missing_indices) != len(keys): @@ -927,12 +975,11 @@ class KVCacheStoreSendingThread(KVTransferThread): addrs.extend(group_addrs) sizes.extend(group_sizes) - # parent_block_hash chains live within a group, not across. if self.enable_kv_event: - prev_key_per_group: dict[int, Any] = {} new_block_hashes = [ maybe_convert_block_hash(bh) for bh in kv_event_block_hashes ] + token_ids_end = token_ids_start + len(event_token_ids or ()) for idx, (s, e, g_idx) in enumerate( zip(starts, ends, group_indices, strict=True) @@ -940,13 +987,24 @@ class KVCacheStoreSendingThread(KVTransferThread): db = self.token_databases[g_idx] if self.enable_kv_event: token_ids = ( - req_meta.token_ids[s:e] - if req_meta.token_ids is not None - else None + event_token_ids[s - token_ids_start : e - token_ids_start] + if event_token_ids is not None + and token_ids_start <= s + and e <= token_ids_end + else [] ) stored_event = BlockStored( block_hashes=[new_block_hashes[idx]], - parent_block_hash=prev_key_per_group.get(g_idx), + # Derive the direct predecessor from the unfiltered + # request chain. Adjacent PUTs need not be adjacent in + # that chain after Store dedup, masks, or TP striding. + parent_block_hash=( + maybe_convert_block_hash( + req_meta.block_hashes[s // db.hash_block_size - 1] + ) + if s > 0 + else None + ), token_ids=token_ids, block_size=db.block_size, lora_id=None, @@ -955,7 +1013,6 @@ class KVCacheStoreSendingThread(KVTransferThread): group_idx=g_idx, ) stored_events.append(stored_event) - prev_key_per_group[g_idx] = new_block_hashes[idx] if current_event is not None: current_event.synchronize() @@ -984,6 +1041,13 @@ class KVCacheStoreSendingThread(KVTransferThread): ) if failed: failed_codes = set(res[i] for i in failed) + if self.enable_kv_event: + failed_indices = set(failed) + stored_events = [ + event + for i, event in enumerate(stored_events) + if i not in failed_indices + ] logger.warning( "batch_put failed: %d/%d keys failed " "(codes=%s, batch_bytes=%d, num_keys=%d), " @@ -1008,6 +1072,7 @@ class KVCacheStoreSendingThread(KVTransferThread): ) else: self._record_saved(req_meta, token_len) + save_completed = True if self._clear_store_pressure(): logger.info( "Mooncake CPU/disk offloading pressure cleared " @@ -1023,10 +1088,18 @@ class KVCacheStoreSendingThread(KVTransferThread): num_failed_keys=len(keys), ) logger.error("Failed to put key %s, error: %s", keys, e) + stored_events.clear() if self.enable_kv_event and stored_events: self.update_kv_event(stored_events) finally: + if self.enable_kv_event and token_len: + self._update_retry_token_ids( + req_meta, + save_completed, + token_ids_start, + event_token_ids, + ) self.finish_store_job(req_meta) self.request_queue.task_done() @@ -1274,15 +1347,19 @@ class MooncakeStoreWorker: self.dcp_rank = get_dcp_group().rank_in_group if self.dcp_size > 1 else 0 assert vllm_config.kv_transfer_config is not None - self.kv_role = vllm_config.kv_transfer_config.kv_role - self.load_async = vllm_config.kv_transfer_config.kv_connector_extra_config.get( - "load_async", True + kv_role = vllm_config.kv_transfer_config.kv_role + assert kv_role is not None + self.kv_role = kv_role + extra_config = vllm_config.kv_transfer_config.kv_connector_extra_config + self.can_put = self.kv_role in ("kv_producer", "kv_both") or ( + extra_config.get("save_decode_cache", False) ) + self.load_async = extra_config.get("load_async", True) # Mirrors MooncakeStoreConnector._capacity_only. - self._capacity_only = self.kv_role == "kv_consumer" and not ( - vllm_config.kv_transfer_config.kv_connector_extra_config.get( - "enable_lookup", True - ) + self._capacity_only = ( + self.kv_role == "kv_consumer" + and not extra_config.get("enable_lookup", True) + and not self.can_put ) self.cache_config = vllm_config.cache_config self.block_size, self.hash_block_size = resolve_kv_cache_block_sizes( @@ -1294,11 +1371,6 @@ class MooncakeStoreWorker: # Initialize MooncakeDistributedStore with its own TransferEngine store_config = MooncakeStoreConfig.load_from_config() - extra_config = ( - vllm_config.kv_transfer_config.kv_connector_extra_config - if vllm_config.kv_transfer_config - else {} - ) self.store = MooncakeDistributedStore() local_ip = get_ip() local_hostname = rdma_utils.get_requester_local_hostname(local_ip) @@ -1617,7 +1689,7 @@ class MooncakeStoreWorker: db.set_block_len(block_lens) # Start transfer threads - if self.kv_role in ["kv_producer", "kv_both"]: + if self.can_put: ready_event_sending = threading.Event() self.kv_send_thread = KVCacheStoreSendingThread( self.store, @@ -1700,7 +1772,7 @@ class MooncakeStoreWorker: assert self.load_async, "load_async must be True for better performance." # Issue stores with CUDA event synchronization. - if self.kv_role in ["kv_producer", "kv_both"]: + if self.can_put: current_event = None for request in meta.requests: if request.can_save: @@ -1714,8 +1786,6 @@ class MooncakeStoreWorker: request.current_event = current_event assert self.kv_send_thread is not None self.kv_send_thread.add_request(request) - - if self.kv_role in ["kv_producer", "kv_both"]: self._close_ended_store_requests(finished_req_ids, meta) # Blocks read by a store job are released by the scheduler when the job