From dc9ae4b8ac2331991ad7091812ef82ece4f8fdc2 Mon Sep 17 00:00:00 2001 From: Guanyi Chen <939416532@qq.com> Date: Mon, 17 Aug 2026 07:09:46 +0800 Subject: [PATCH] [Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372) Signed-off-by: Guanyi Chen <939416532@qq.com> --- .../unit/test_mooncake_store_hma_e2e.py | 15 +- .../unit/test_mooncake_store_scheduler.py | 179 +++++++++----- .../unit/test_mooncake_store_worker.py | 222 ++++++++++-------- .../v1/mooncake/store/connector.py | 22 +- .../kv_connector/v1/mooncake/store/data.py | 24 +- .../v1/mooncake/store/scheduler.py | 104 ++++++-- .../kv_connector/v1/mooncake/store/worker.py | 164 ++++++++----- 7 files changed, 477 insertions(+), 253 deletions(-) diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py index 6c8af7fb73..32457f13d5 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_hma_e2e.py @@ -228,12 +228,11 @@ def test_e2e_swa_plus_full_save_then_lookup_hits(): block_ids=([0, 1, 2, 3], [0, 1, 2, 3]), block_hashes=hs, can_save=True, + store_job_id=1, ) - send_thread.add_stored_request("r0") - # Put the request in the queue so task_done() doesn't underflow. - send_thread.request_queue.put(save_req) - req = send_thread.request_queue.get() - send_thread._handle_request(req) + # add_request also queues the job, so task_done() doesn't underflow. + send_thread.add_request(save_req) + send_thread._handle_request(send_thread.request_queue.get()) # Point worker.store at the dict store (the worker constructor captured # the MagicMock; replace with the real dict store for lookup). @@ -499,15 +498,15 @@ def test_offload_syncs_event_before_put(): can_save=True, num_prompt_tokens=12, partial_tail_offloads=[(1, 7, 12)], + store_job_id=1, ) req.current_event = event - send.add_stored_request("r1") - send.request_queue.put(req) + send.add_request(req) send._handle_request(send.request_queue.get()) assert send.request_queue.qsize() == 0 assert store._data - assert send.stored_requests["r1"] == 0 + assert send.stored_requests["r1"] == set() event.synchronize.assert_called_once() diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py index 13960c4034..998a8408e6 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_scheduler.py @@ -5,12 +5,14 @@ from types import SimpleNamespace from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( LoadSpec, + MooncakeStoreWorkerMetadata, ReqMeta, RequestTracker, ) from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import ( MooncakeStoreScheduler, ) +from vllm.v1.core.block_pool import BlockPool def _make_bare_scheduler( @@ -27,6 +29,12 @@ def _make_bare_scheduler( scheduler._unfinished_request_ids = {"req-0"} scheduler._unfinished_requests = {} scheduler._request_trackers = {} + scheduler._gpu_block_pool = BlockPool( + num_gpu_blocks=64, enable_caching=True, hash_block_size=hash_block_size + ) + scheduler._num_workers = 1 + scheduler._next_store_job_id = 0 + scheduler._pinned_saves = {} return scheduler @@ -64,6 +72,14 @@ def _make_preemption_scheduler_output(): ) +def _make_worker_output(completed_saves: dict[int, int]) -> SimpleNamespace: + return SimpleNamespace( + kv_connector_worker_meta=MooncakeStoreWorkerMetadata( + completed_saves=completed_saves + ) + ) + + def _add_unfinished_request( scheduler: MooncakeStoreScheduler, *, @@ -133,7 +149,7 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap(): assert tracker.num_saved_tokens == 48 -def test_preemption_resets_tracker_before_request_finished(): +def test_preemption_resets_tracker(): scheduler = _make_bare_scheduler() _add_unfinished_request( scheduler, @@ -152,8 +168,6 @@ def test_preemption_resets_tracker_before_request_finished(): assert tracker.token_ids is None assert tracker.has_pending_offload is False assert tracker.prefill_end_tokens == 0 - request = SimpleNamespace(request_id="req-0") - assert scheduler.request_finished(request, ([0, 1],)) == (False, None) def test_preemption_clears_stale_load_state(): @@ -215,7 +229,7 @@ def test_pending_load_does_not_co_queue_save(): # enqueue a save in the same scheduling step. Co-queuing both produces a # recv+send pair for the same req_id, and the scheduler's # _update_from_kv_xfer_finished then trips `assert req_id in self.requests` - # when both completions land for the delay-freed request. + # when a completion lands for a request it has already dropped. scheduler = _make_bare_scheduler() _make_pending_load_unfinished_request( scheduler, @@ -238,9 +252,7 @@ def test_pending_load_does_not_co_queue_save(): # Load is still issued as planned. assert req_meta.load_spec is not None assert req_meta.load_spec.can_load is True - # And the tracker's saved-tokens watermark stays at 0 so request_finished - # later sees `num_saved_tokens <= 0` and frees immediately rather than - # waiting for a finished_sending that will never come. + # And the save watermark does not advance for a save that was never queued. tracker = scheduler._request_trackers["req-0"] assert tracker.num_saved_tokens == 0 @@ -661,30 +673,34 @@ def test_disabled_lookup_reports_no_hit_without_querying_client(): assert scheduler.load_specs == {} -def test_pending_partial_tail_emits_offload_only_reqmeta(): - # A sub-block prompt never produces a block-aligned save, so the partial- - # tail offload arriving this step is emitted as an offload-only ReqMeta - # (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so - # the worker skips the normal save). Pending-offload state delays the free - # without advancing the normal-save watermark before the put succeeds. - scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) +def _add_pending_partial_tail_request( + scheduler: MooncakeStoreScheduler, + *, + num_tokens: int, + block_hashes: list[bytes], + block_ids: tuple[list[int], ...], +) -> SimpleNamespace: + """Register a sub-block request and return the step that offloads its tail. + + The CoW block holding the tail is block 7, which the core deliberately keeps + out of the request's block table. + """ request = SimpleNamespace( - all_token_ids=list(range(12)), - block_hashes=[b"h0", b"h1", b"h2"], + all_token_ids=list(range(num_tokens)), + block_hashes=block_hashes, num_output_placeholders=0, num_prompt_tokens=12, ) - scheduler._unfinished_requests["req-0"] = (request, ([0],)) + scheduler._unfinished_requests["req-0"] = (request, block_ids) scheduler._request_trackers["req-0"] = RequestTracker( req_id="req-0", - token_len=12, - allocated_block_ids=([0],), + token_len=num_tokens, + allocated_block_ids=block_ids, num_saved_tokens=0, - token_ids=list(range(12)), - prefill_end_tokens=12, + token_ids=list(range(num_tokens)), + prefill_end_tokens=num_tokens, ) - - out = SimpleNamespace( + return SimpleNamespace( finished_req_ids=set(), preempted_req_ids=set(), scheduled_new_reqs=[], @@ -699,6 +715,21 @@ def test_pending_partial_tail_emits_offload_only_reqmeta(): partial_tail_offloads={"req-0": [(1, 7, 12)]}, ) + +def test_pending_partial_tail_emits_offload_only_reqmeta(): + # A sub-block prompt never produces a block-aligned save, so the partial- + # tail offload arriving this step is emitted as an offload-only ReqMeta + # (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so + # the worker skips the normal save), without advancing the normal-save + # watermark before the put succeeds. + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + out = _add_pending_partial_tail_request( + scheduler, + num_tokens=12, + block_hashes=[b"h0", b"h1", b"h2"], + block_ids=([0],), + ) + meta = scheduler.build_connector_meta(out) assert len(meta.requests) == 1 @@ -712,42 +743,16 @@ def test_pending_partial_tail_emits_offload_only_reqmeta(): tracker = scheduler._request_trackers["req-0"] assert tracker.num_saved_tokens == 0 assert tracker.has_pending_offload is True - request = SimpleNamespace(request_id="req-0") - assert scheduler.request_finished(request, ([0],)) == (True, None) def test_resumed_partial_tail_uses_handoff_boundary(): scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) - request = SimpleNamespace( - all_token_ids=list(range(20)), + # Resumption replays prompt + previously generated tokens. + out = _add_pending_partial_tail_request( + scheduler, + num_tokens=20, block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"], - num_output_placeholders=0, - num_prompt_tokens=12, - ) - scheduler._unfinished_requests["req-0"] = (request, ([0, 1],)) - scheduler._request_trackers["req-0"] = RequestTracker( - req_id="req-0", - token_len=20, - allocated_block_ids=([0, 1],), - num_saved_tokens=0, - token_ids=list(range(20)), - # Resumption replays prompt + previously generated tokens. - prefill_end_tokens=20, - ) - - out = SimpleNamespace( - finished_req_ids=set(), - preempted_req_ids=set(), - scheduled_new_reqs=[], - scheduled_cached_reqs=SimpleNamespace( - req_ids=[], - new_block_ids=[], - num_computed_tokens=[], - resumed_req_ids=set(), - ), - num_scheduled_tokens={}, - scheduled_spec_decode_tokens={}, - partial_tail_offloads={"req-0": [(1, 7, 12)]}, + block_ids=([0, 1],), ) meta = scheduler.build_connector_meta(out) @@ -791,3 +796,69 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary(): tracker = scheduler._request_trackers["req-0"] assert tracker.num_saved_tokens == 48 assert tracker.has_pending_offload is True + + +def test_partial_tail_cow_block_is_referenced_for_the_job(): + # The CoW block a partial-tail offload reads is deliberately kept out of the + # request block table, so it is absent from ReqMeta.block_ids. The worker + # DMAs out of it just as asynchronously, so it needs its own reference. + scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True) + out = _add_pending_partial_tail_request( + scheduler, + num_tokens=12, + block_hashes=[b"h0", b"h1", b"h2"], + block_ids=([0],), + ) + pool = scheduler._gpu_block_pool + + meta = scheduler.build_connector_meta(out) + + store_job_id = meta.requests[0].store_job_id + # It leads the list, as in `pop_blocks_for_free`, so that the reversed free + # puts it last in eviction priority. + assert scheduler._pinned_saves[store_job_id][0] == [7, 0] + assert pool.blocks[7].ref_cnt == 1 + + scheduler.update_connector_output(_make_worker_output({store_job_id: 1})) + assert pool.blocks[7].ref_cnt == 0 + + +def test_store_job_blocks_are_released_once_every_rank_reports(): + # Every rank DMAs the job's blocks on its own, so the reference can only be + # dropped once the last of them reports. Until then the engine has to keep + # stepping: a completion only reaches the scheduler as worker metadata + # attached to a step, and a finishing request no longer defers its own free. + scheduler = _make_bare_scheduler() + scheduler._num_workers = 2 + _add_unfinished_request( + scheduler, + token_ids=list(range(48)), + block_hashes=[b"h0", b"h1", b"h2"], + prefill_end_tokens=48, + ) + pool = scheduler._gpu_block_pool + assert scheduler.has_pending_push_work() is False + + meta = scheduler.build_connector_meta( + _make_scheduler_output(scheduled_spec_tokens=None) + ) + store_job_id = meta.requests[0].store_job_id + assert pool.blocks[2].ref_cnt == 1 + assert scheduler.has_pending_push_work() is True + + scheduler.update_connector_output(_make_worker_output({store_job_id: 1})) + assert pool.blocks[2].ref_cnt == 1 + assert scheduler.has_pending_push_work() is True + + scheduler.update_connector_output(_make_worker_output({store_job_id: 1})) + assert pool.blocks[2].ref_cnt == 0 + assert scheduler.has_pending_push_work() is False + + +def test_worker_metadata_aggregates_completions_across_ranks(): + # The engine merges each rank's metadata before the scheduler sees it, so a + # job that every rank finished in one step arrives as a single count. + merged = MooncakeStoreWorkerMetadata(completed_saves={1: 1}).aggregate( + MooncakeStoreWorkerMetadata(completed_saves={1: 1, 2: 1}) + ) + assert merged.completed_saves == {1: 2, 2: 1} diff --git a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py index 034297100a..4655c5716e 100644 --- a/tests/v1/kv_connector/unit/test_mooncake_store_worker.py +++ b/tests/v1/kv_connector/unit/test_mooncake_store_worker.py @@ -1,6 +1,8 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import contextlib +import itertools import json import logging import math @@ -156,6 +158,17 @@ def _make_load_req( ) +_TEST_SAVE_SEQ = itertools.count(1) + + +def _run_store_req(thread, req_meta: ReqMeta) -> None: + """Register, enqueue and run a store job the way the worker does.""" + if req_meta.store_job_id is None: + req_meta.store_job_id = next(_TEST_SAVE_SEQ) + thread.add_request(req_meta) + thread._handle_request(thread.request_queue.get()) + + def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta: return ReqMeta( req_id=req_id, @@ -385,27 +398,23 @@ def test_store_sending_thread_skips_request_during_cpu_pressure(): ] thread = _make_store_sending_thread(store) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert thread._store_pressure_active is True assert "req-a" in thread._skip_store_requests assert store.batch_put_from_multi_buffers.call_count == 1 - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"])) + _run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"])) assert store.batch_put_from_multi_buffers.call_count == 1 - thread.add_stored_request("req-b") - thread._handle_request(_make_store_req("req-b", [b"b0", b"b1"])) + _run_store_req(thread, _make_store_req("req-b", [b"b0", b"b1"])) assert thread._store_pressure_active is False assert "req-a" not in thread._skip_store_requests assert store.batch_put_from_multi_buffers.call_count == 2 - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a4", b"a5"])) + _run_store_req(thread, _make_store_req("req-a", [b"a4", b"a5"])) assert store.batch_put_from_multi_buffers.call_count == 3 @@ -418,8 +427,7 @@ def test_store_sending_thread_records_mooncake_metrics(): stats = MooncakeStoreConnectorStats() thread._record_operation_cb = stats.record_operation - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert len(stats.data["save_exists"]) == 1 assert stats.data["save_exists"][0]["num_keys"] == 2 @@ -497,16 +505,16 @@ def test_store_sending_thread_delta_saves_only_new_full_attention_chunks(): store.batch_put_from_multi_buffers.return_value = [256, 256] thread = _make_store_sending_thread(store) - thread.add_stored_request("req-a") thread._saved_offset["req-a"] = 32 - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=64, block_ids=([0, 1, 2, 3],), block_hashes=[b"a0", b"a1", b"a2", b"a3"], can_save=True, - ) + ), ) keys = store.batch_is_exist.call_args.args[0] @@ -523,16 +531,16 @@ def test_store_sending_thread_delta_strides_with_local_phase(): store.batch_put_from_multi_buffers.return_value = [256] thread = _make_store_sending_thread(store, tp_rank=0, put_step=2) - thread.add_stored_request("req-a") thread._saved_offset["req-a"] = 16 - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=64, block_ids=([0, 1, 2, 3],), block_hashes=[b"a0", b"a1", b"a2", b"a3"], can_save=True, - ) + ), ) keys = store.batch_is_exist.call_args.args[0] @@ -550,15 +558,15 @@ def test_tp_sharded_group_saves_every_block_on_every_rank(): thread = _make_store_sending_thread(store, tp_rank=0, put_step=2) thread.group_put_steps = [1] - thread.add_stored_request("req-a") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=64, block_ids=([0, 1, 2, 3],), block_hashes=[b"a0", b"a1", b"a2", b"a3"], can_save=True, - ) + ), ) keys = store.batch_is_exist.call_args.args[0] @@ -576,15 +584,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure(): thread._store_pressure_active = True thread._skip_store_requests.add("req-a") - thread.add_stored_request("req-a") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=16, block_ids=([0],), block_hashes=[b"a0"], can_save=True, - ) + ), ) store.batch_is_exist.assert_not_called() @@ -595,15 +603,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure(): # The next batch resumes from offset 0, re-covering the chunk skipped under # pressure (chunk 0) rather than losing it. - thread.add_stored_request("req-a") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=64, block_ids=([0, 1, 2, 3],), block_hashes=[b"a0", b"a1", b"a2", b"a3"], can_save=True, - ) + ), ) keys = store.batch_is_exist.call_args.args[0] @@ -703,13 +711,11 @@ def test_partial_tail_offload_honors_active_pressure_gate(): thread = _make_partial_tail_send_thread(store) thread._store_pressure_active = True thread._skip_store_requests.add("req-a") - thread.add_stored_request("req-a") - thread._handle_request(_make_partial_tail_req([1, 2, 3])) + _run_store_req(thread, _make_partial_tail_req([1, 2, 3])) store.batch_is_exist.assert_not_called() store.batch_put_from_multi_buffers.assert_not_called() - assert thread.stored_requests["req-a"] == 0 def test_partial_tail_put_failure_activates_pressure_gate(): @@ -717,17 +723,14 @@ def test_partial_tail_put_failure_activates_pressure_gate(): store.batch_is_exist.side_effect = lambda keys: [0] * len(keys) store.batch_put_from_multi_buffers.return_value = [256, -200, 256] thread = _make_partial_tail_send_thread(store) - thread.add_stored_request("req-a") - thread._handle_request(_make_partial_tail_req([1, 2, 3])) + _run_store_req(thread, _make_partial_tail_req([1, 2, 3])) assert thread._store_pressure_active is True assert thread._skip_store_requests == {"req-a"} assert thread._saved_offset.get("req-a", 0) == 0 - assert thread.stored_requests["req-a"] == 0 - thread.add_stored_request("req-a") - thread._handle_request(_make_partial_tail_req([1, 2, 3])) + _run_store_req(thread, _make_partial_tail_req([1, 2, 3])) assert store.batch_put_from_multi_buffers.call_count == 1 @@ -737,16 +740,16 @@ def test_store_sending_thread_delta_start_rank_saves_second_local_chunk(): store.batch_put_from_multi_buffers.return_value = [256, 256] thread = _make_store_sending_thread(store, tp_rank=1, put_step=2) - thread.add_stored_request("req-a") thread._saved_offset["req-a"] = 16 - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=64, block_ids=([0, 1, 2, 3],), block_hashes=[b"a0", b"a1", b"a2", b"a3"], can_save=True, - ) + ), ) keys = store.batch_is_exist.call_args.args[0] @@ -790,16 +793,16 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks(): token_databases=[db_full, db_masked], ) - thread.add_stored_request("req-a") thread._saved_offset["req-a"] = 32 - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=64, block_ids=([0, 1, 2, 3], [0, 1, 2, 3]), block_hashes=[b"a0", b"a1", b"a2", b"a3"], can_save=True, - ) + ), ) keys = store.batch_is_exist.call_args.args[0] @@ -845,15 +848,15 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group(): coord=coord, token_databases=[db0, db1], ) - thread.add_stored_request("req-a") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="req-a", token_len_chunk=48, block_ids=([0, 1, 2], [2, 1, 0]), block_hashes=[b"a0", b"a1", b"a2"], can_save=True, - ) + ), ) db0.prepare_value.assert_not_called() @@ -881,46 +884,71 @@ def test_store_sending_thread_only_skips_on_no_available_handle(): ] thread = _make_store_sending_thread(store) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert thread._store_pressure_active is False assert "req-a" not in thread._skip_store_requests assert store.batch_put_from_multi_buffers.call_count == 1 - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"])) + _run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"])) assert store.batch_put_from_multi_buffers.call_count == 2 -def test_store_sending_thread_releases_pin_on_batch_is_exist_failure(): - # `batch_is_exist` raising must still decrement `stored_requests` so the - # scheduler can drop `delay_free_blocks` and release the pinned GPU blocks. - store = MagicMock() - store.batch_is_exist.side_effect = RuntimeError("mooncake down") - thread = _make_store_sending_thread(store) - - thread.add_stored_request("req-a") - with pytest.raises(RuntimeError): - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) - - assert thread.stored_requests["req-a"] == 0 - store.batch_put_from_multi_buffers.assert_not_called() - - -def test_store_sending_thread_releases_pin_on_batch_put_failure(): - # `batch_put_from_multi_buffers` raising is logged (not re-raised), and the - # pin must still be released through the finally block. +@pytest.mark.parametrize( + "failing_call", ["batch_is_exist", "batch_put_from_multi_buffers"] +) +def test_store_sending_thread_reports_job_when_store_raises(failing_call): + # A store that blows up must still report its job, or the scheduler keeps + # the job's GPU block references for the rest of the run. store = MagicMock() store.batch_is_exist.return_value = [0, 0] - store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error") + getattr(store, failing_call).side_effect = RuntimeError("mooncake down") thread = _make_store_sending_thread(store) + req = _make_store_req("req-a", [b"a0", b"a1"]) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + with contextlib.suppress(RuntimeError): + _run_store_req(thread, req) - assert thread.stored_requests["req-a"] == 0 + assert thread.take_completed_saves() == {req.store_job_id: 1} + + +def test_store_sending_thread_reports_job_when_the_preamble_raises(): + # The report has to survive a failure before the first store call too, so + # every dequeue leaves through the same exit. + thread = _make_store_sending_thread(MagicMock()) + req = _make_store_req("req-a", [b"a0", b"a1"]) + req.token_len_chunk = None # type: ignore[assignment] + + with contextlib.suppress(TypeError): + _run_store_req(thread, req) + + assert thread.take_completed_saves() == {req.store_job_id: 1} + thread.request_queue.task_done.assert_called_once() + + +def test_stale_store_job_cannot_touch_a_reused_request_id(): + # A preempted request resumes under its original id, so a job left over from + # the retired generation carries a req_id that now belongs to a live one. + thread = _make_store_sending_thread(MagicMock()) + stale = _make_store_req("req-a", [b"a0", b"a1"]) + stale.store_job_id = 1 + thread.add_request(stale) + thread._record_saved(stale, 32) + thread.delete_finished_stored_request("req-a") + + live = _make_store_req("req-a", [b"a0", b"a1"]) + live.store_job_id = 2 + thread.add_request(live) + + thread.finish_store_job(stale) + thread._record_saved(stale, 64) + thread._mark_request_skipped_for_pressure(stale) + + assert thread.is_live_store_job(live) + assert not thread.is_live_store_job(stale) + assert thread._saved_offset.get("req-a") is None + assert "req-a" not in thread._skip_store_requests def test_store_recving_thread_reports_failed_block_ids(): @@ -991,8 +1019,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set replicate_config = SimpleNamespace(preferred_segment="10.0.0.7:50053") thread = _make_store_sending_thread(store, replicate_config=replicate_config) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert store.batch_put_from_multi_buffers.call_count == 1 call_args = store.batch_put_from_multi_buffers.call_args.args @@ -1010,8 +1037,7 @@ def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_ replicate_config = SimpleNamespace() thread = _make_store_sending_thread(store, replicate_config=replicate_config) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert store.batch_put_from_multi_buffers.call_count == 1 call_args = store.batch_put_from_multi_buffers.call_args.args @@ -1067,8 +1093,7 @@ def test_store_sending_thread_sets_group_ids_when_enabled(): supports_group_ids=True, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert store.batch_put_from_multi_buffers.call_count == 1 keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args @@ -1092,8 +1117,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_flag_disabled(): supports_group_ids=True, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert store.batch_put_from_multi_buffers.call_count == 1 assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config @@ -1112,8 +1136,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_unsupported(): supports_group_ids=False, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) assert store.batch_put_from_multi_buffers.call_count == 1 assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config @@ -1180,8 +1203,7 @@ def test_store_sending_thread_group_id_excludes_physical_sharding(): supports_group_ids=True, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args assert keys == [ @@ -1210,8 +1232,7 @@ def test_store_sending_thread_multiple_segments_share_logical_group_id(): supports_group_ids=True, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args assert keys == [ @@ -1260,8 +1281,7 @@ def test_store_sending_thread_group_ids_share_across_kv_cache_groups(): supports_group_ids=True, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_multi_group_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_multi_group_store_req("req-a", [b"a0", b"a1"])) keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args assert keys == [ @@ -1296,8 +1316,7 @@ def test_store_sending_thread_group_ids_follow_missing_key_filter(): supports_group_ids=True, ) - thread.add_stored_request("req-a") - thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"])) + _run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"])) keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args assert keys == ["test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131"] @@ -1866,15 +1885,15 @@ def test_store_sending_thread_clamps_token_len_to_lcm(): # token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk). thread = _make_store_sending_thread(store) - thread.add_stored_request("r0") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="r0", token_len_chunk=33, block_ids=([0, 1, 2],), block_hashes=[b"a0", b"a1", b"a2"], can_save=True, - ) + ), ) keys = store.batch_put_from_multi_buffers.call_args.args[0] @@ -1905,20 +1924,19 @@ def test_store_sending_thread_skips_when_token_len_below_lcm(): store, coord=coord, token_databases=[db], block_size=64 ) - thread.add_stored_request("r0") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="r0", token_len_chunk=32, block_ids=([0, 1],), block_hashes=[b"a0", b"a1"], can_save=True, - ) + ), ) store.batch_is_exist.assert_not_called() store.batch_put_from_multi_buffers.assert_not_called() - assert thread.stored_requests["r0"] == 0 def test_store_sending_thread_only_stores_swa_blocks_in_window(): @@ -1982,15 +2000,15 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window(): ) hs = [bytes([i + 1]) * 4 for i in range(8)] - thread.add_stored_request("r0") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="r0", token_len_chunk=64, block_ids=([0, 1], list(range(8))), block_hashes=hs, can_save=True, - ) + ), ) keys = store.batch_put_from_multi_buffers.call_args.args[0] @@ -2057,16 +2075,16 @@ def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks(): ) hs = [bytes([i + 1]) * 4 for i in range(8)] - thread.add_stored_request("r0") thread._saved_offset["r0"] = 32 - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="r0", token_len_chunk=64, block_ids=([0, 1], list(range(8))), block_hashes=hs, can_save=True, - ) + ), ) keys = store.batch_put_from_multi_buffers.call_args.args[0] @@ -2128,8 +2146,8 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata(): thread.enable_kv_event = True hs = [bytes([i + 1]) * 4 for i in range(4)] - thread.add_stored_request("r0") - thread._handle_request( + _run_store_req( + thread, ReqMeta( req_id="r0", token_len_chunk=32, @@ -2137,7 +2155,7 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata(): block_hashes=hs, can_save=True, token_ids=list(range(32)), - ) + ), ) full_event, swa_event = thread.get_kv_events() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py index ca32c3345c..297fc22ad8 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/connector.py @@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorBase_V1, KVConnectorMetadata, KVConnectorRole, + KVConnectorWorkerMetadata, SupportsHMA, ) from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( @@ -37,6 +38,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import ( from vllm.forward_context import ForwardContext from vllm.logger import init_logger from vllm.v1.attention.backend import AttentionMetadata +from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.sched.output import SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig @@ -201,6 +203,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): request, blocks, num_external_tokens ) + def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None: + assert self.connector_scheduler is not None + self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool) + + def has_pending_push_work(self) -> bool: + assert self.connector_scheduler is not None + return self.connector_scheduler.has_pending_push_work() + def build_connector_meta( self, scheduler_output: SchedulerOutput, @@ -208,6 +218,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): assert self.connector_scheduler is not None return self.connector_scheduler.build_connector_meta(scheduler_output) + def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None: + assert self.connector_worker is not None + return self.connector_worker.build_connector_worker_meta() + def request_finished( self, request: Request, @@ -220,8 +234,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): request: Request, block_ids: tuple[list[int], ...], ) -> tuple[bool, dict[str, Any] | None]: - assert self.connector_scheduler is not None - return self.connector_scheduler.request_finished(request, block_ids) + # An in-flight store job holds its own reference on the blocks it reads, + # so a finishing request never has to defer freeing them. + return False, None def reset_cache(self) -> bool | None: """Reset the external Mooncake store on prefix-cache reset. @@ -241,6 +256,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA): return None def update_connector_output(self, connector_output: KVConnectorOutput): + assert self.connector_scheduler is not None + self.connector_scheduler.update_connector_output(connector_output) + kv_cache_events = connector_output.kv_cache_events if not kv_cache_events or not isinstance( kv_cache_events, MooncakeStoreKVEvents diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py index 6daa1e82ea..55ffc040cf 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/data.py @@ -6,7 +6,7 @@ """Data classes for MooncakeStoreConnector.""" from collections.abc import Iterable, Sequence -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import cast import numpy as np @@ -14,6 +14,7 @@ import torch from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, + KVConnectorWorkerMetadata, ) from vllm.logger import init_logger from vllm.utils.math_utils import cdiv @@ -366,6 +367,10 @@ class ReqMeta: token_ids: list[int] | None = None num_prompt_tokens: int | None = None + # Identifies this store job for the engine's lifetime. A request id cannot + # serve that purpose: it is reused once a preempted request resumes, so it + # would release the wrong job's blocks. + store_job_id: int | None = None # Core-provided per-mamba-group # (group_id, cow_block_id, boundary_tokens) for this request's partial tail. # Present only on the producer's CoW step; drives the connector's offload @@ -434,6 +439,23 @@ class ReqMeta: ) +@dataclass +class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata): + """Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job.""" + + completed_saves: dict[int, int] = field(default_factory=dict) + + def aggregate( + self, other: "KVConnectorWorkerMetadata" + ) -> "MooncakeStoreWorkerMetadata": + assert isinstance(other, MooncakeStoreWorkerMetadata) + for store_job_id, count in other.completed_saves.items(): + self.completed_saves[store_job_id] = ( + self.completed_saves.get(store_job_id, 0) + count + ) + return self + + class MooncakeStoreConnectorMetadata(KVConnectorMetadata): """Metadata passed from scheduler to worker.""" diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py index 42c6f3fa99..f583b3fcfd 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/scheduler.py @@ -5,8 +5,6 @@ # (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/). """Scheduler-side logic for MooncakeStoreConnector.""" -from typing import Any - from vllm.config import VllmConfig from vllm.distributed.kv_transfer.kv_connector.v1.base import ( KVConnectorMetadata, @@ -17,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501 LoadSpec, MooncakeStoreConnectorMetadata, + MooncakeStoreWorkerMetadata, ReqMeta, RequestTracker, ) @@ -24,10 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import ( LookupKeyClient, ) from vllm.logger import init_logger +from vllm.v1.core.block_pool import BlockPool from vllm.v1.core.kv_cache_manager import KVCacheBlocks from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput from vllm.v1.kv_cache_interface import KVCacheConfig +from vllm.v1.outputs import KVConnectorOutput from vllm.v1.request import Request logger = init_logger(__name__) @@ -78,6 +79,15 @@ class MooncakeStoreScheduler: self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {} self._unfinished_request_ids: set[str] = set() + self._gpu_block_pool: BlockPool | None = None + self._num_workers = vllm_config.parallel_config.world_size + self._next_store_job_id = 0 + # store_job_id -> (referenced block ids, ranks yet to report completion) + self._pinned_saves: dict[int, tuple[list[int], int]] = {} + + def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None: + self._gpu_block_pool = gpu_block_pool + def get_num_new_matched_tokens( self, request: Request, @@ -392,33 +402,75 @@ class MooncakeStoreScheduler: ) ) + self._reference_save_blocks(meta) return meta - def request_finished( - self, - request: Request, - block_ids: tuple[list[int], ...], - ) -> tuple[bool, dict[str, Any] | None]: - """Determine whether to delay freeing blocks for async save.""" - if self.kv_role == "kv_consumer": - return False, None - tracker = self._request_trackers.get(request.request_id) - # Missing tracker can happen when the request is aborted before the - # connector observes the normal finished lifecycle or is preempted - # before finishing. - if tracker is None or ( - tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload - ): - return False, None - total_blocks = sum(len(g) for g in block_ids) - delay_free_blocks = total_blocks > 0 - if delay_free_blocks: - logger.debug( - "Delaying free of %d blocks for request %s", - total_blocks, - request.request_id, + def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None: + """Take a GPU block reference for every store job this step emits. + + The worker DMAs out of these blocks after the step that scheduled them, + so a reference keeps them out of the free queue even once the request + itself is freed, until every rank reports the job done. + """ + pool = self._gpu_block_pool + for req_meta in meta.requests: + if not req_meta.can_save: + continue + assert pool is not None, ( + "GPU block pool must be bound before any store job is emitted" ) - return delay_free_blocks, None + req_meta.store_job_id = store_job_id = self._next_store_job_id + self._next_store_job_id += 1 + block_ids: list[int] = [] + if req_meta.partial_tail_offloads: + # A partial-tail CoW block is deliberately kept out of the + # request's block table, so it is absent from `block_ids` even + # though the worker DMAs out of it just as asynchronously. + # It leads the list, as in `pop_blocks_for_free`. + block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads] + # Every allocated block is referenced, not just the ones covering + # this job's token range: a rank resumes from its own last + # successful offset, which lags the scheduler's whenever a save was + # skipped or failed, so it may read anywhere below the range. + block_ids += [bid for group in req_meta.block_ids for bid in group] + if not block_ids: + continue + self._pinned_saves[store_job_id] = (block_ids, self._num_workers) + pool.touch([pool.blocks[bid] for bid in block_ids]) + + def update_connector_output(self, connector_output: KVConnectorOutput) -> None: + """Drop the block references of store jobs every rank has finished.""" + meta = connector_output.kv_connector_worker_meta + if not isinstance(meta, MooncakeStoreWorkerMetadata): + return + pool = self._gpu_block_pool + assert pool is not None + for store_job_id, count in meta.completed_saves.items(): + pinned = self._pinned_saves.get(store_job_id) + if pinned is None: + # The job referenced no blocks, so nothing was recorded for it. + continue + block_ids, remaining = pinned + remaining -= count + if remaining > 0: + self._pinned_saves[store_job_id] = (block_ids, remaining) + continue + assert remaining == 0, ( + f"store job {store_job_id} reported by too many ranks" + ) + del self._pinned_saves[store_job_id] + # Tail-first, as elsewhere, so the shared prefix is evicted last. + pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids)) + + def has_pending_push_work(self) -> bool: + """Keep the engine stepping while any store job still holds block refs. + + Completions only reach the scheduler as worker metadata on a step, so an + engine that quiesced with jobs in flight would leave those references + held indefinitely. Nothing else keeps it alive now that a finishing + request no longer defers its own free. + """ + return bool(self._pinned_saves) def reset_store(self) -> bool: """Trigger a global ``remove_all(force=True)`` on the Mooncake master. diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py index 762e5eba92..e25aec3fbd 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/mooncake/store/worker.py @@ -17,7 +17,6 @@ import queue import socket import threading import time -from collections import defaultdict from collections.abc import Callable, Sequence from concurrent.futures import Future, ThreadPoolExecutor from dataclasses import dataclass @@ -46,6 +45,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( ChunkedTokenDatabase, KeyMetadata, MooncakeStoreConnectorMetadata, + MooncakeStoreWorkerMetadata, PoolKey, ReqMeta, ) @@ -506,7 +506,16 @@ class KVCacheStoreSendingThread(KVTransferThread): self.group_put_steps = group_put_steps self.coord = coord self.kv_role = kv_role - self.stored_requests: defaultdict[str, int] = defaultdict(int) + # req_id -> ids of its store jobs that are still queued or running. + # Keying by store_job_id, which never repeats for the engine's lifetime, + # rather than counting jobs per request id makes the ledger immune to id + # reuse across preemption: a job left over from a retired generation is + # missing from the set its resumed generation builds, so it can no longer + # retire that generation, rewind its resume offset, or mark it skipped. + self.stored_requests: dict[str, set[int]] = {} + # store_job_id -> times this rank finished with it, drained every step + # so the scheduler can release the blocks it referenced for those jobs. + self._completed_saves: dict[int, int] = {} self.enable_kv_event = enable_kv_event # Caller always passes a non-None ReplicateConfig — see # MooncakeStoreWorker.__init__ where store_replicate_config is built. @@ -522,14 +531,20 @@ class KVCacheStoreSendingThread(KVTransferThread): # batch resumes here, so pressure-skipped or failed ranges are retried. self._saved_offset: dict[str, int] = {} - def add_stored_request(self, req_id: str): + def add_request(self, request: ReqMeta) -> None: + # Register before enqueueing so a job is never picked up unledgered. + assert request.store_job_id is not None with self.done_task_lock: - self.stored_requests[req_id] += 1 + self.stored_requests.setdefault(request.req_id, set()).add( + request.store_job_id + ) + super().add_request(request) - def dec_stored_request(self, req_id: str): + def is_live_store_job(self, req_meta: ReqMeta) -> bool: with self.done_task_lock: - if req_id in self.stored_requests: - self.stored_requests[req_id] -= 1 + return req_meta.store_job_id in self.stored_requests.get( + req_meta.req_id, () + ) def delete_finished_stored_request(self, req_id: str): with self.done_task_lock: @@ -538,21 +553,51 @@ class KVCacheStoreSendingThread(KVTransferThread): self._skip_store_requests.discard(req_id) self._saved_offset.pop(req_id, None) - def _record_saved(self, req_id: str, token_len: int) -> None: - # Guard on liveness so a concurrent finish/preempt pop isn't recreated. + def finish_store_job(self, req_meta: ReqMeta) -> None: + """Retire a job from the ledger and report its blocks as no longer read. + + Every path out of a job must reach this, skips and failures included: a + job that never reports leaves its blocks referenced for the rest of the + run. The discard is a no-op for a job whose generation already retired. + """ + store_job_id = req_meta.store_job_id + assert store_job_id is not None, ( + "a queued store job always carries a store_job_id" + ) with self.done_task_lock: - if req_id in self.stored_requests: - self._saved_offset[req_id] = token_len + live = self.stored_requests.get(req_meta.req_id) + if live is not None: + live.discard(store_job_id) + self._completed_saves[store_job_id] = ( + self._completed_saves.get(store_job_id, 0) + 1 + ) + + def take_completed_saves(self) -> dict[int, int]: + with self.done_task_lock: + completed = self._completed_saves + self._completed_saves = {} + return completed + + def _record_saved(self, req_meta: ReqMeta, token_len: int) -> None: + # Guard on job liveness so neither a concurrent finish/preempt pop nor a + # stale job's offset is written back over the live generation's. + with self.done_task_lock: + if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()): + self._saved_offset[req_meta.req_id] = token_len def _should_skip_request(self, req_id: str) -> bool: with self.done_task_lock: return self._store_pressure_active and req_id in self._skip_store_requests - def _mark_request_skipped_for_pressure(self, req_id: str) -> bool: + def _mark_request_skipped_for_pressure(self, req_meta: ReqMeta) -> bool: + req_id = req_meta.req_id with self.done_task_lock: already_skipped = req_id in self._skip_store_requests self._store_pressure_active = True - self._skip_store_requests.add(req_id) + # The pressure itself is global, but only a live job may sentence its + # own request to being skipped. + if req_meta.store_job_id in self.stored_requests.get(req_id, ()): + self._skip_store_requests.add(req_id) return already_skipped def _clear_store_pressure(self) -> bool: @@ -727,7 +772,7 @@ class KVCacheStoreSendingThread(KVTransferThread): failed_codes, ) if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes: - self._mark_request_skipped_for_pressure(req_meta.req_id) + self._mark_request_skipped_for_pressure(req_meta) return False if self._clear_store_pressure(): @@ -738,22 +783,20 @@ class KVCacheStoreSendingThread(KVTransferThread): return True def _handle_request(self, req_meta: ReqMeta): - # Cache hits are always a multiple of ``lcm_block_size`` tokens, which - # is also ``store_mask``'s precondition. - lcm_block_size = self.coord.lcm_block_size - token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size - block_ids_per_group = req_meta.block_ids - req_id = req_meta.req_id - current_event = req_meta.current_event - - if req_id not in self.stored_requests: - self.request_queue.task_done() - return - - # Decrement the in-flight counter and signal task_done() in `finally` - # so the scheduler can release the GPU blocks it pinned for this - # request (via `delay_free_blocks`) even when the store path raises. + # The single `finally` is the only way out, so the scheduler releases + # this job's GPU block references however the job ends. try: + # Cache hits are always a multiple of ``lcm_block_size`` tokens, + # which is also ``store_mask``'s precondition. + lcm_block_size = self.coord.lcm_block_size + token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size + block_ids_per_group = req_meta.block_ids + req_id = req_meta.req_id + current_event = req_meta.current_event + + if not self.is_live_store_job(req_meta): + return + if self._should_skip_request(req_id): logger.debug( "Skipping Mooncake store for request %s while CPU/disk " @@ -808,7 +851,7 @@ class KVCacheStoreSendingThread(KVTransferThread): group_indices.append(g_idx) if not keys: - self._record_saved(req_id, token_len) + self._record_saved(req_meta, token_len) return # Check which blocks already exist (dedup) @@ -834,7 +877,7 @@ class KVCacheStoreSendingThread(KVTransferThread): ] if not missing_indices: - self._record_saved(req_id, token_len) + self._record_saved(req_meta, token_len) return if len(missing_indices) != len(keys): @@ -954,7 +997,7 @@ class KVCacheStoreSendingThread(KVTransferThread): ) if ( MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes - and not self._mark_request_skipped_for_pressure(req_id) + and not self._mark_request_skipped_for_pressure(req_meta) ): logger.warning( "Detected Mooncake CPU/disk offloading pressure " @@ -964,7 +1007,7 @@ class KVCacheStoreSendingThread(KVTransferThread): req_id, ) else: - self._record_saved(req_id, token_len) + self._record_saved(req_meta, token_len) if self._clear_store_pressure(): logger.info( "Mooncake CPU/disk offloading pressure cleared " @@ -984,7 +1027,7 @@ class KVCacheStoreSendingThread(KVTransferThread): if self.enable_kv_event and stored_events: self.update_kv_event(stored_events) finally: - self.dec_stored_request(req_id) + self.finish_store_job(req_meta) self.request_queue.task_done() @@ -1670,16 +1713,15 @@ class MooncakeStoreWorker: continue request.current_event = current_event assert self.kv_send_thread is not None - self.kv_send_thread.add_stored_request(request.req_id) self.kv_send_thread.add_request(request) - # Check completion of previously queued transfers - done_sending = ( - self._get_and_clear_finished_sending(finished_req_ids, meta) - if self.kv_role in ["kv_producer", "kv_both"] - else set() - ) + if self.kv_role in ["kv_producer", "kv_both"]: + self._close_ended_store_requests(finished_req_ids, meta) + # Blocks read by a store job are released by the scheduler when the job + # reports back (see build_connector_worker_meta), so no request ever waits + # on a `finished_sending` signal to get its blocks back. + done_sending: set[str] = set() done_recving: set[str] = set() if self.load_async: for recv_thread in self.kv_recv_threads: @@ -1727,35 +1769,37 @@ class MooncakeStoreWorker: self.kv_connector_stats = MooncakeStoreConnectorStats() return kv_connector_stats - def _get_and_clear_finished_sending( + def _close_ended_store_requests( self, finished_req_ids: set[str], meta: MooncakeStoreConnectorMetadata, - ) -> set[str]: + ) -> None: + """Retire the ledger entries of requests that finished or were preempted. + + An entry may only go once its jobs have drained, because they still read + the resume offset it owns; a request that comes back after preemption + then saves from the start rather than from where the last attempt got to. + """ assert self.kv_send_thread is not None - finished_sending: set[str] = set() for req_id in meta.preempted_req_ids: self.kv_send_thread.delete_finished_stored_request(req_id) - for req_id in self.kv_send_thread.stored_requests.copy(): - if ( - self.kv_send_thread.stored_requests[req_id] == 0 - and req_id in self.finished_store_req - ): - self.finished_store_req.remove(req_id) - finished_sending.add(req_id) - self.kv_send_thread.delete_finished_stored_request(req_id) - - for req_id in finished_req_ids: - req_remain_jobs = self.kv_send_thread.stored_requests.get(req_id) - if req_remain_jobs == 0: - finished_sending.add(req_id) - self.kv_send_thread.delete_finished_stored_request(req_id) - elif req_remain_jobs is not None: + for req_id in finished_req_ids | self.finished_store_req: + if self.kv_send_thread.stored_requests.get(req_id): + # Queued jobs still need the resume offset; retire on a later step. self.finished_store_req.add(req_id) + else: + self.finished_store_req.discard(req_id) + self.kv_send_thread.delete_finished_stored_request(req_id) - return finished_sending + def build_connector_worker_meta(self) -> MooncakeStoreWorkerMetadata | None: + if self.kv_send_thread is None: + return None + completed_saves = self.kv_send_thread.take_completed_saves() + if not completed_saves: + return None + return MooncakeStoreWorkerMetadata(completed_saves=completed_saves) def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int: """Check how many prefix tokens exist in the store.