[Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372)

Signed-off-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
Guanyi Chen
2026-08-16 16:09:46 -07:00
committed by GitHub
parent eee538d5da
commit dc9ae4b8ac
7 changed files with 477 additions and 253 deletions
@@ -228,12 +228,11 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=hs,
can_save=True,
store_job_id=1,
)
send_thread.add_stored_request("r0")
# Put the request in the queue so task_done() doesn't underflow.
send_thread.request_queue.put(save_req)
req = send_thread.request_queue.get()
send_thread._handle_request(req)
# add_request also queues the job, so task_done() doesn't underflow.
send_thread.add_request(save_req)
send_thread._handle_request(send_thread.request_queue.get())
# Point worker.store at the dict store (the worker constructor captured
# the MagicMock; replace with the real dict store for lookup).
@@ -499,15 +498,15 @@ def test_offload_syncs_event_before_put():
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, 7, 12)],
store_job_id=1,
)
req.current_event = event
send.add_stored_request("r1")
send.request_queue.put(req)
send.add_request(req)
send._handle_request(send.request_queue.get())
assert send.request_queue.qsize() == 0
assert store._data
assert send.stored_requests["r1"] == 0
assert send.stored_requests["r1"] == set()
event.synchronize.assert_called_once()
@@ -5,12 +5,14 @@ from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
)
from vllm.v1.core.block_pool import BlockPool
def _make_bare_scheduler(
@@ -27,6 +29,12 @@ def _make_bare_scheduler(
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
scheduler._request_trackers = {}
scheduler._gpu_block_pool = BlockPool(
num_gpu_blocks=64, enable_caching=True, hash_block_size=hash_block_size
)
scheduler._num_workers = 1
scheduler._next_store_job_id = 0
scheduler._pinned_saves = {}
return scheduler
@@ -64,6 +72,14 @@ def _make_preemption_scheduler_output():
)
def _make_worker_output(completed_saves: dict[int, int]) -> SimpleNamespace:
return SimpleNamespace(
kv_connector_worker_meta=MooncakeStoreWorkerMetadata(
completed_saves=completed_saves
)
)
def _add_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
@@ -133,7 +149,7 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
assert tracker.num_saved_tokens == 48
def test_preemption_resets_tracker_before_request_finished():
def test_preemption_resets_tracker():
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
@@ -152,8 +168,6 @@ def test_preemption_resets_tracker_before_request_finished():
assert tracker.token_ids is None
assert tracker.has_pending_offload is False
assert tracker.prefill_end_tokens == 0
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
def test_preemption_clears_stale_load_state():
@@ -215,7 +229,7 @@ def test_pending_load_does_not_co_queue_save():
# enqueue a save in the same scheduling step. Co-queuing both produces a
# recv+send pair for the same req_id, and the scheduler's
# _update_from_kv_xfer_finished then trips `assert req_id in self.requests`
# when both completions land for the delay-freed request.
# when a completion lands for a request it has already dropped.
scheduler = _make_bare_scheduler()
_make_pending_load_unfinished_request(
scheduler,
@@ -238,9 +252,7 @@ def test_pending_load_does_not_co_queue_save():
# Load is still issued as planned.
assert req_meta.load_spec is not None
assert req_meta.load_spec.can_load is True
# And the tracker's saved-tokens watermark stays at 0 so request_finished
# later sees `num_saved_tokens <= 0` and frees immediately rather than
# waiting for a finished_sending that will never come.
# And the save watermark does not advance for a save that was never queued.
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
@@ -661,30 +673,34 @@ def test_disabled_lookup_reports_no_hit_without_querying_client():
assert scheduler.load_specs == {}
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
# the worker skips the normal save). Pending-offload state delays the free
# without advancing the normal-save watermark before the put succeeds.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
def _add_pending_partial_tail_request(
scheduler: MooncakeStoreScheduler,
*,
num_tokens: int,
block_hashes: list[bytes],
block_ids: tuple[list[int], ...],
) -> SimpleNamespace:
"""Register a sub-block request and return the step that offloads its tail.
The CoW block holding the tail is block 7, which the core deliberately keeps
out of the request's block table.
"""
request = SimpleNamespace(
all_token_ids=list(range(12)),
block_hashes=[b"h0", b"h1", b"h2"],
all_token_ids=list(range(num_tokens)),
block_hashes=block_hashes,
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0],))
scheduler._unfinished_requests["req-0"] = (request, block_ids)
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=12,
allocated_block_ids=([0],),
token_len=num_tokens,
allocated_block_ids=block_ids,
num_saved_tokens=0,
token_ids=list(range(12)),
prefill_end_tokens=12,
token_ids=list(range(num_tokens)),
prefill_end_tokens=num_tokens,
)
out = SimpleNamespace(
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
@@ -699,6 +715,21 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
partial_tail_offloads={"req-0": [(1, 7, 12)]},
)
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
# the worker skips the normal save), without advancing the normal-save
# watermark before the put succeeds.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([0],),
)
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
@@ -712,42 +743,16 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0],)) == (True, None)
def test_resumed_partial_tail_uses_handoff_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(20)),
# Resumption replays prompt + previously generated tokens.
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=20,
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=20,
allocated_block_ids=([0, 1],),
num_saved_tokens=0,
token_ids=list(range(20)),
# Resumption replays prompt + previously generated tokens.
prefill_end_tokens=20,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
block_ids=([0, 1],),
)
meta = scheduler.build_connector_meta(out)
@@ -791,3 +796,69 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 48
assert tracker.has_pending_offload is True
def test_partial_tail_cow_block_is_referenced_for_the_job():
# The CoW block a partial-tail offload reads is deliberately kept out of the
# request block table, so it is absent from ReqMeta.block_ids. The worker
# DMAs out of it just as asynchronously, so it needs its own reference.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([0],),
)
pool = scheduler._gpu_block_pool
meta = scheduler.build_connector_meta(out)
store_job_id = meta.requests[0].store_job_id
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
# puts it last in eviction priority.
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
assert pool.blocks[7].ref_cnt == 1
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert pool.blocks[7].ref_cnt == 0
def test_store_job_blocks_are_released_once_every_rank_reports():
# Every rank DMAs the job's blocks on its own, so the reference can only be
# dropped once the last of them reports. Until then the engine has to keep
# stepping: a completion only reaches the scheduler as worker metadata
# attached to a step, and a finishing request no longer defers its own free.
scheduler = _make_bare_scheduler()
scheduler._num_workers = 2
_add_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
prefill_end_tokens=48,
)
pool = scheduler._gpu_block_pool
assert scheduler.has_pending_push_work() is False
meta = scheduler.build_connector_meta(
_make_scheduler_output(scheduled_spec_tokens=None)
)
store_job_id = meta.requests[0].store_job_id
assert pool.blocks[2].ref_cnt == 1
assert scheduler.has_pending_push_work() is True
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert pool.blocks[2].ref_cnt == 1
assert scheduler.has_pending_push_work() is True
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert pool.blocks[2].ref_cnt == 0
assert scheduler.has_pending_push_work() is False
def test_worker_metadata_aggregates_completions_across_ranks():
# The engine merges each rank's metadata before the scheduler sees it, so a
# job that every rank finished in one step arrives as a single count.
merged = MooncakeStoreWorkerMetadata(completed_saves={1: 1}).aggregate(
MooncakeStoreWorkerMetadata(completed_saves={1: 1, 2: 1})
)
assert merged.completed_saves == {1: 2, 2: 1}
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import itertools
import json
import logging
import math
@@ -156,6 +158,17 @@ def _make_load_req(
)
_TEST_SAVE_SEQ = itertools.count(1)
def _run_store_req(thread, req_meta: ReqMeta) -> None:
"""Register, enqueue and run a store job the way the worker does."""
if req_meta.store_job_id is None:
req_meta.store_job_id = next(_TEST_SAVE_SEQ)
thread.add_request(req_meta)
thread._handle_request(thread.request_queue.get())
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
return ReqMeta(
req_id=req_id,
@@ -385,27 +398,23 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert thread._store_pressure_active is True
assert "req-a" in thread._skip_store_requests
assert store.batch_put_from_multi_buffers.call_count == 1
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
assert store.batch_put_from_multi_buffers.call_count == 1
thread.add_stored_request("req-b")
thread._handle_request(_make_store_req("req-b", [b"b0", b"b1"]))
_run_store_req(thread, _make_store_req("req-b", [b"b0", b"b1"]))
assert thread._store_pressure_active is False
assert "req-a" not in thread._skip_store_requests
assert store.batch_put_from_multi_buffers.call_count == 2
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a4", b"a5"]))
_run_store_req(thread, _make_store_req("req-a", [b"a4", b"a5"]))
assert store.batch_put_from_multi_buffers.call_count == 3
@@ -418,8 +427,7 @@ def test_store_sending_thread_records_mooncake_metrics():
stats = MooncakeStoreConnectorStats()
thread._record_operation_cb = stats.record_operation
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert len(stats.data["save_exists"]) == 1
assert stats.data["save_exists"][0]["num_keys"] == 2
@@ -497,16 +505,16 @@ def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -523,16 +531,16 @@ def test_store_sending_thread_delta_strides_with_local_phase():
store.batch_put_from_multi_buffers.return_value = [256]
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -550,15 +558,15 @@ def test_tp_sharded_group_saves_every_block_on_every_rank():
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
thread.group_put_steps = [1]
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -576,15 +584,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=16,
block_ids=([0],),
block_hashes=[b"a0"],
can_save=True,
)
),
)
store.batch_is_exist.assert_not_called()
@@ -595,15 +603,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
# The next batch resumes from offset 0, re-covering the chunk skipped under
# pressure (chunk 0) rather than losing it.
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -703,13 +711,11 @@ def test_partial_tail_offload_honors_active_pressure_gate():
thread = _make_partial_tail_send_thread(store)
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["req-a"] == 0
def test_partial_tail_put_failure_activates_pressure_gate():
@@ -717,17 +723,14 @@ def test_partial_tail_put_failure_activates_pressure_gate():
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
thread = _make_partial_tail_send_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
assert thread._store_pressure_active is True
assert thread._skip_store_requests == {"req-a"}
assert thread._saved_offset.get("req-a", 0) == 0
assert thread.stored_requests["req-a"] == 0
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
assert store.batch_put_from_multi_buffers.call_count == 1
@@ -737,16 +740,16 @@ def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -790,16 +793,16 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
token_databases=[db_full, db_masked],
)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -845,15 +848,15 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
coord=coord,
token_databases=[db0, db1],
)
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=48,
block_ids=([0, 1, 2], [2, 1, 0]),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
)
),
)
db0.prepare_value.assert_not_called()
@@ -881,46 +884,71 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert thread._store_pressure_active is False
assert "req-a" not in thread._skip_store_requests
assert store.batch_put_from_multi_buffers.call_count == 1
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
assert store.batch_put_from_multi_buffers.call_count == 2
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
# `batch_is_exist` raising must still decrement `stored_requests` so the
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
store = MagicMock()
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
with pytest.raises(RuntimeError):
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert thread.stored_requests["req-a"] == 0
store.batch_put_from_multi_buffers.assert_not_called()
def test_store_sending_thread_releases_pin_on_batch_put_failure():
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
# pin must still be released through the finally block.
@pytest.mark.parametrize(
"failing_call", ["batch_is_exist", "batch_put_from_multi_buffers"]
)
def test_store_sending_thread_reports_job_when_store_raises(failing_call):
# A store that blows up must still report its job, or the scheduler keeps
# the job's GPU block references for the rest of the run.
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
getattr(store, failing_call).side_effect = RuntimeError("mooncake down")
thread = _make_store_sending_thread(store)
req = _make_store_req("req-a", [b"a0", b"a1"])
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
with contextlib.suppress(RuntimeError):
_run_store_req(thread, req)
assert thread.stored_requests["req-a"] == 0
assert thread.take_completed_saves() == {req.store_job_id: 1}
def test_store_sending_thread_reports_job_when_the_preamble_raises():
# The report has to survive a failure before the first store call too, so
# every dequeue leaves through the same exit.
thread = _make_store_sending_thread(MagicMock())
req = _make_store_req("req-a", [b"a0", b"a1"])
req.token_len_chunk = None # type: ignore[assignment]
with contextlib.suppress(TypeError):
_run_store_req(thread, req)
assert thread.take_completed_saves() == {req.store_job_id: 1}
thread.request_queue.task_done.assert_called_once()
def test_stale_store_job_cannot_touch_a_reused_request_id():
# A preempted request resumes under its original id, so a job left over from
# the retired generation carries a req_id that now belongs to a live one.
thread = _make_store_sending_thread(MagicMock())
stale = _make_store_req("req-a", [b"a0", b"a1"])
stale.store_job_id = 1
thread.add_request(stale)
thread._record_saved(stale, 32)
thread.delete_finished_stored_request("req-a")
live = _make_store_req("req-a", [b"a0", b"a1"])
live.store_job_id = 2
thread.add_request(live)
thread.finish_store_job(stale)
thread._record_saved(stale, 64)
thread._mark_request_skipped_for_pressure(stale)
assert thread.is_live_store_job(live)
assert not thread.is_live_store_job(stale)
assert thread._saved_offset.get("req-a") is None
assert "req-a" not in thread._skip_store_requests
def test_store_recving_thread_reports_failed_block_ids():
@@ -991,8 +1019,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set
replicate_config = SimpleNamespace(preferred_segment="10.0.0.7:50053")
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
call_args = store.batch_put_from_multi_buffers.call_args.args
@@ -1010,8 +1037,7 @@ def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_
replicate_config = SimpleNamespace()
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
call_args = store.batch_put_from_multi_buffers.call_args.args
@@ -1067,8 +1093,7 @@ def test_store_sending_thread_sets_group_ids_when_enabled():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
@@ -1092,8 +1117,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_flag_disabled():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
@@ -1112,8 +1136,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_unsupported():
supports_group_ids=False,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
@@ -1180,8 +1203,7 @@ def test_store_sending_thread_group_id_excludes_physical_sharding():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1210,8 +1232,7 @@ def test_store_sending_thread_multiple_segments_share_logical_group_id():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1260,8 +1281,7 @@ def test_store_sending_thread_group_ids_share_across_kv_cache_groups():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_multi_group_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_multi_group_store_req("req-a", [b"a0", b"a1"]))
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1296,8 +1316,7 @@ def test_store_sending_thread_group_ids_follow_missing_key_filter():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == ["test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131"]
@@ -1866,15 +1885,15 @@ def test_store_sending_thread_clamps_token_len_to_lcm():
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
thread = _make_store_sending_thread(store)
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=33,
block_ids=([0, 1, 2],),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
)
),
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
@@ -1905,20 +1924,19 @@ def test_store_sending_thread_skips_when_token_len_below_lcm():
store, coord=coord, token_databases=[db], block_size=64
)
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=32,
block_ids=([0, 1],),
block_hashes=[b"a0", b"a1"],
can_save=True,
)
),
)
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["r0"] == 0
def test_store_sending_thread_only_stores_swa_blocks_in_window():
@@ -1982,15 +2000,15 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
)
),
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
@@ -2057,16 +2075,16 @@ def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._saved_offset["r0"] = 32
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
)
),
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
@@ -2128,8 +2146,8 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
thread.enable_kv_event = True
hs = [bytes([i + 1]) * 4 for i in range(4)]
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=32,
@@ -2137,7 +2155,7 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
block_hashes=hs,
can_save=True,
token_ids=list(range(32)),
)
),
)
full_event, swa_event = thread.get_kv_events()
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorBase_V1,
KVConnectorMetadata,
KVConnectorRole,
KVConnectorWorkerMetadata,
SupportsHMA,
)
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
@@ -37,6 +38,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
from vllm.forward_context import ForwardContext
from vllm.logger import init_logger
from vllm.v1.attention.backend import AttentionMetadata
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.sched.output import SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
@@ -201,6 +203,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
request, blocks, num_external_tokens
)
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
assert self.connector_scheduler is not None
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
def has_pending_push_work(self) -> bool:
assert self.connector_scheduler is not None
return self.connector_scheduler.has_pending_push_work()
def build_connector_meta(
self,
scheduler_output: SchedulerOutput,
@@ -208,6 +218,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
assert self.connector_scheduler is not None
return self.connector_scheduler.build_connector_meta(scheduler_output)
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
assert self.connector_worker is not None
return self.connector_worker.build_connector_worker_meta()
def request_finished(
self,
request: Request,
@@ -220,8 +234,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
assert self.connector_scheduler is not None
return self.connector_scheduler.request_finished(request, block_ids)
# An in-flight store job holds its own reference on the blocks it reads,
# so a finishing request never has to defer freeing them.
return False, None
def reset_cache(self) -> bool | None:
"""Reset the external Mooncake store on prefix-cache reset.
@@ -241,6 +256,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
return None
def update_connector_output(self, connector_output: KVConnectorOutput):
assert self.connector_scheduler is not None
self.connector_scheduler.update_connector_output(connector_output)
kv_cache_events = connector_output.kv_cache_events
if not kv_cache_events or not isinstance(
kv_cache_events, MooncakeStoreKVEvents
@@ -6,7 +6,7 @@
"""Data classes for MooncakeStoreConnector."""
from collections.abc import Iterable, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from typing import cast
import numpy as np
@@ -14,6 +14,7 @@ import torch
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
KVConnectorWorkerMetadata,
)
from vllm.logger import init_logger
from vllm.utils.math_utils import cdiv
@@ -366,6 +367,10 @@ class ReqMeta:
token_ids: list[int] | None = None
num_prompt_tokens: int | None = None
# Identifies this store job for the engine's lifetime. A request id cannot
# serve that purpose: it is reused once a preempted request resumes, so it
# would release the wrong job's blocks.
store_job_id: int | None = None
# Core-provided per-mamba-group
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
# Present only on the producer's CoW step; drives the connector's offload
@@ -434,6 +439,23 @@ class ReqMeta:
)
@dataclass
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
completed_saves: dict[int, int] = field(default_factory=dict)
def aggregate(
self, other: "KVConnectorWorkerMetadata"
) -> "MooncakeStoreWorkerMetadata":
assert isinstance(other, MooncakeStoreWorkerMetadata)
for store_job_id, count in other.completed_saves.items():
self.completed_saves[store_job_id] = (
self.completed_saves.get(store_job_id, 0) + count
)
return self
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
"""Metadata passed from scheduler to worker."""
@@ -5,8 +5,6 @@
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
"""Scheduler-side logic for MooncakeStoreConnector."""
from typing import Any
from vllm.config import VllmConfig
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
KVConnectorMetadata,
@@ -17,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
LoadSpec,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
@@ -24,10 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
LookupKeyClient,
)
from vllm.logger import init_logger
from vllm.v1.core.block_pool import BlockPool
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
from vllm.v1.kv_cache_interface import KVCacheConfig
from vllm.v1.outputs import KVConnectorOutput
from vllm.v1.request import Request
logger = init_logger(__name__)
@@ -78,6 +79,15 @@ class MooncakeStoreScheduler:
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
self._unfinished_request_ids: set[str] = set()
self._gpu_block_pool: BlockPool | None = None
self._num_workers = vllm_config.parallel_config.world_size
self._next_store_job_id = 0
# store_job_id -> (referenced block ids, ranks yet to report completion)
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
self._gpu_block_pool = gpu_block_pool
def get_num_new_matched_tokens(
self,
request: Request,
@@ -392,33 +402,75 @@ class MooncakeStoreScheduler:
)
)
self._reference_save_blocks(meta)
return meta
def request_finished(
self,
request: Request,
block_ids: tuple[list[int], ...],
) -> tuple[bool, dict[str, Any] | None]:
"""Determine whether to delay freeing blocks for async save."""
if self.kv_role == "kv_consumer":
return False, None
tracker = self._request_trackers.get(request.request_id)
# Missing tracker can happen when the request is aborted before the
# connector observes the normal finished lifecycle or is preempted
# before finishing.
if tracker is None or (
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
):
return False, None
total_blocks = sum(len(g) for g in block_ids)
delay_free_blocks = total_blocks > 0
if delay_free_blocks:
logger.debug(
"Delaying free of %d blocks for request %s",
total_blocks,
request.request_id,
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
"""Take a GPU block reference for every store job this step emits.
The worker DMAs out of these blocks after the step that scheduled them,
so a reference keeps them out of the free queue even once the request
itself is freed, until every rank reports the job done.
"""
pool = self._gpu_block_pool
for req_meta in meta.requests:
if not req_meta.can_save:
continue
assert pool is not None, (
"GPU block pool must be bound before any store job is emitted"
)
return delay_free_blocks, None
req_meta.store_job_id = store_job_id = self._next_store_job_id
self._next_store_job_id += 1
block_ids: list[int] = []
if req_meta.partial_tail_offloads:
# A partial-tail CoW block is deliberately kept out of the
# request's block table, so it is absent from `block_ids` even
# though the worker DMAs out of it just as asynchronously.
# It leads the list, as in `pop_blocks_for_free`.
block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads]
# Every allocated block is referenced, not just the ones covering
# this job's token range: a rank resumes from its own last
# successful offset, which lags the scheduler's whenever a save was
# skipped or failed, so it may read anywhere below the range.
block_ids += [bid for group in req_meta.block_ids for bid in group]
if not block_ids:
continue
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
pool.touch([pool.blocks[bid] for bid in block_ids])
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
"""Drop the block references of store jobs every rank has finished."""
meta = connector_output.kv_connector_worker_meta
if not isinstance(meta, MooncakeStoreWorkerMetadata):
return
pool = self._gpu_block_pool
assert pool is not None
for store_job_id, count in meta.completed_saves.items():
pinned = self._pinned_saves.get(store_job_id)
if pinned is None:
# The job referenced no blocks, so nothing was recorded for it.
continue
block_ids, remaining = pinned
remaining -= count
if remaining > 0:
self._pinned_saves[store_job_id] = (block_ids, remaining)
continue
assert remaining == 0, (
f"store job {store_job_id} reported by too many ranks"
)
del self._pinned_saves[store_job_id]
# Tail-first, as elsewhere, so the shared prefix is evicted last.
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
def has_pending_push_work(self) -> bool:
"""Keep the engine stepping while any store job still holds block refs.
Completions only reach the scheduler as worker metadata on a step, so an
engine that quiesced with jobs in flight would leave those references
held indefinitely. Nothing else keeps it alive now that a finishing
request no longer defers its own free.
"""
return bool(self._pinned_saves)
def reset_store(self) -> bool:
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
@@ -17,7 +17,6 @@ import queue
import socket
import threading
import time
from collections import defaultdict
from collections.abc import Callable, Sequence
from concurrent.futures import Future, ThreadPoolExecutor
from dataclasses import dataclass
@@ -46,6 +45,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
ChunkedTokenDatabase,
KeyMetadata,
MooncakeStoreConnectorMetadata,
MooncakeStoreWorkerMetadata,
PoolKey,
ReqMeta,
)
@@ -506,7 +506,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
self.group_put_steps = group_put_steps
self.coord = coord
self.kv_role = kv_role
self.stored_requests: defaultdict[str, int] = defaultdict(int)
# req_id -> ids of its store jobs that are still queued or running.
# Keying by store_job_id, which never repeats for the engine's lifetime,
# rather than counting jobs per request id makes the ledger immune to id
# reuse across preemption: a job left over from a retired generation is
# missing from the set its resumed generation builds, so it can no longer
# retire that generation, rewind its resume offset, or mark it skipped.
self.stored_requests: dict[str, set[int]] = {}
# store_job_id -> times this rank finished with it, drained every step
# so the scheduler can release the blocks it referenced for those jobs.
self._completed_saves: dict[int, int] = {}
self.enable_kv_event = enable_kv_event
# Caller always passes a non-None ReplicateConfig — see
# MooncakeStoreWorker.__init__ where store_replicate_config is built.
@@ -522,14 +531,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
# batch resumes here, so pressure-skipped or failed ranges are retried.
self._saved_offset: dict[str, int] = {}
def add_stored_request(self, req_id: str):
def add_request(self, request: ReqMeta) -> None:
# Register before enqueueing so a job is never picked up unledgered.
assert request.store_job_id is not None
with self.done_task_lock:
self.stored_requests[req_id] += 1
self.stored_requests.setdefault(request.req_id, set()).add(
request.store_job_id
)
super().add_request(request)
def dec_stored_request(self, req_id: str):
def is_live_store_job(self, req_meta: ReqMeta) -> bool:
with self.done_task_lock:
if req_id in self.stored_requests:
self.stored_requests[req_id] -= 1
return req_meta.store_job_id in self.stored_requests.get(
req_meta.req_id, ()
)
def delete_finished_stored_request(self, req_id: str):
with self.done_task_lock:
@@ -538,21 +553,51 @@ class KVCacheStoreSendingThread(KVTransferThread):
self._skip_store_requests.discard(req_id)
self._saved_offset.pop(req_id, None)
def _record_saved(self, req_id: str, token_len: int) -> None:
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
def finish_store_job(self, req_meta: ReqMeta) -> None:
"""Retire a job from the ledger and report its blocks as no longer read.
Every path out of a job must reach this, skips and failures included: a
job that never reports leaves its blocks referenced for the rest of the
run. The discard is a no-op for a job whose generation already retired.
"""
store_job_id = req_meta.store_job_id
assert store_job_id is not None, (
"a queued store job always carries a store_job_id"
)
with self.done_task_lock:
if req_id in self.stored_requests:
self._saved_offset[req_id] = token_len
live = self.stored_requests.get(req_meta.req_id)
if live is not None:
live.discard(store_job_id)
self._completed_saves[store_job_id] = (
self._completed_saves.get(store_job_id, 0) + 1
)
def take_completed_saves(self) -> dict[int, int]:
with self.done_task_lock:
completed = self._completed_saves
self._completed_saves = {}
return completed
def _record_saved(self, req_meta: ReqMeta, token_len: int) -> None:
# Guard on job liveness so neither a concurrent finish/preempt pop nor a
# stale job's offset is written back over the live generation's.
with self.done_task_lock:
if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()):
self._saved_offset[req_meta.req_id] = token_len
def _should_skip_request(self, req_id: str) -> bool:
with self.done_task_lock:
return self._store_pressure_active and req_id in self._skip_store_requests
def _mark_request_skipped_for_pressure(self, req_id: str) -> bool:
def _mark_request_skipped_for_pressure(self, req_meta: ReqMeta) -> bool:
req_id = req_meta.req_id
with self.done_task_lock:
already_skipped = req_id in self._skip_store_requests
self._store_pressure_active = True
self._skip_store_requests.add(req_id)
# The pressure itself is global, but only a live job may sentence its
# own request to being skipped.
if req_meta.store_job_id in self.stored_requests.get(req_id, ()):
self._skip_store_requests.add(req_id)
return already_skipped
def _clear_store_pressure(self) -> bool:
@@ -727,7 +772,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
failed_codes,
)
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
self._mark_request_skipped_for_pressure(req_meta.req_id)
self._mark_request_skipped_for_pressure(req_meta)
return False
if self._clear_store_pressure():
@@ -738,22 +783,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
return True
def _handle_request(self, req_meta: ReqMeta):
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
# is also ``store_mask``'s precondition.
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if req_id not in self.stored_requests:
self.request_queue.task_done()
return
# Decrement the in-flight counter and signal task_done() in `finally`
# so the scheduler can release the GPU blocks it pinned for this
# request (via `delay_free_blocks`) even when the store path raises.
# The single `finally` is the only way out, so the scheduler releases
# this job's GPU block references however the job ends.
try:
# Cache hits are always a multiple of ``lcm_block_size`` tokens,
# which is also ``store_mask``'s precondition.
lcm_block_size = self.coord.lcm_block_size
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
block_ids_per_group = req_meta.block_ids
req_id = req_meta.req_id
current_event = req_meta.current_event
if not self.is_live_store_job(req_meta):
return
if self._should_skip_request(req_id):
logger.debug(
"Skipping Mooncake store for request %s while CPU/disk "
@@ -808,7 +851,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
group_indices.append(g_idx)
if not keys:
self._record_saved(req_id, token_len)
self._record_saved(req_meta, token_len)
return
# Check which blocks already exist (dedup)
@@ -834,7 +877,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
]
if not missing_indices:
self._record_saved(req_id, token_len)
self._record_saved(req_meta, token_len)
return
if len(missing_indices) != len(keys):
@@ -954,7 +997,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
)
if (
MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes
and not self._mark_request_skipped_for_pressure(req_id)
and not self._mark_request_skipped_for_pressure(req_meta)
):
logger.warning(
"Detected Mooncake CPU/disk offloading pressure "
@@ -964,7 +1007,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
req_id,
)
else:
self._record_saved(req_id, token_len)
self._record_saved(req_meta, token_len)
if self._clear_store_pressure():
logger.info(
"Mooncake CPU/disk offloading pressure cleared "
@@ -984,7 +1027,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
if self.enable_kv_event and stored_events:
self.update_kv_event(stored_events)
finally:
self.dec_stored_request(req_id)
self.finish_store_job(req_meta)
self.request_queue.task_done()
@@ -1670,16 +1713,15 @@ class MooncakeStoreWorker:
continue
request.current_event = current_event
assert self.kv_send_thread is not None
self.kv_send_thread.add_stored_request(request.req_id)
self.kv_send_thread.add_request(request)
# Check completion of previously queued transfers
done_sending = (
self._get_and_clear_finished_sending(finished_req_ids, meta)
if self.kv_role in ["kv_producer", "kv_both"]
else set()
)
if self.kv_role in ["kv_producer", "kv_both"]:
self._close_ended_store_requests(finished_req_ids, meta)
# Blocks read by a store job are released by the scheduler when the job
# reports back (see build_connector_worker_meta), so no request ever waits
# on a `finished_sending` signal to get its blocks back.
done_sending: set[str] = set()
done_recving: set[str] = set()
if self.load_async:
for recv_thread in self.kv_recv_threads:
@@ -1727,35 +1769,37 @@ class MooncakeStoreWorker:
self.kv_connector_stats = MooncakeStoreConnectorStats()
return kv_connector_stats
def _get_and_clear_finished_sending(
def _close_ended_store_requests(
self,
finished_req_ids: set[str],
meta: MooncakeStoreConnectorMetadata,
) -> set[str]:
) -> None:
"""Retire the ledger entries of requests that finished or were preempted.
An entry may only go once its jobs have drained, because they still read
the resume offset it owns; a request that comes back after preemption
then saves from the start rather than from where the last attempt got to.
"""
assert self.kv_send_thread is not None
finished_sending: set[str] = set()
for req_id in meta.preempted_req_ids:
self.kv_send_thread.delete_finished_stored_request(req_id)
for req_id in self.kv_send_thread.stored_requests.copy():
if (
self.kv_send_thread.stored_requests[req_id] == 0
and req_id in self.finished_store_req
):
self.finished_store_req.remove(req_id)
finished_sending.add(req_id)
self.kv_send_thread.delete_finished_stored_request(req_id)
for req_id in finished_req_ids:
req_remain_jobs = self.kv_send_thread.stored_requests.get(req_id)
if req_remain_jobs == 0:
finished_sending.add(req_id)
self.kv_send_thread.delete_finished_stored_request(req_id)
elif req_remain_jobs is not None:
for req_id in finished_req_ids | self.finished_store_req:
if self.kv_send_thread.stored_requests.get(req_id):
# Queued jobs still need the resume offset; retire on a later step.
self.finished_store_req.add(req_id)
else:
self.finished_store_req.discard(req_id)
self.kv_send_thread.delete_finished_stored_request(req_id)
return finished_sending
def build_connector_worker_meta(self) -> MooncakeStoreWorkerMetadata | None:
if self.kv_send_thread is None:
return None
completed_saves = self.kv_send_thread.take_completed_saves()
if not completed_saves:
return None
return MooncakeStoreWorkerMetadata(completed_saves=completed_saves)
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
"""Check how many prefix tokens exist in the store.