[Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372)
Signed-off-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
@@ -228,12 +228,11 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
store_job_id=1,
|
||||
)
|
||||
send_thread.add_stored_request("r0")
|
||||
# Put the request in the queue so task_done() doesn't underflow.
|
||||
send_thread.request_queue.put(save_req)
|
||||
req = send_thread.request_queue.get()
|
||||
send_thread._handle_request(req)
|
||||
# add_request also queues the job, so task_done() doesn't underflow.
|
||||
send_thread.add_request(save_req)
|
||||
send_thread._handle_request(send_thread.request_queue.get())
|
||||
|
||||
# Point worker.store at the dict store (the worker constructor captured
|
||||
# the MagicMock; replace with the real dict store for lookup).
|
||||
@@ -499,15 +498,15 @@ def test_offload_syncs_event_before_put():
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
store_job_id=1,
|
||||
)
|
||||
req.current_event = event
|
||||
send.add_stored_request("r1")
|
||||
|
||||
send.request_queue.put(req)
|
||||
send.add_request(req)
|
||||
send._handle_request(send.request_queue.get())
|
||||
assert send.request_queue.qsize() == 0
|
||||
assert store._data
|
||||
assert send.stored_requests["r1"] == 0
|
||||
assert send.stored_requests["r1"] == set()
|
||||
event.synchronize.assert_called_once()
|
||||
|
||||
|
||||
|
||||
@@ -5,12 +5,14 @@ from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
LoadSpec,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
|
||||
|
||||
def _make_bare_scheduler(
|
||||
@@ -27,6 +29,12 @@ def _make_bare_scheduler(
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
scheduler._gpu_block_pool = BlockPool(
|
||||
num_gpu_blocks=64, enable_caching=True, hash_block_size=hash_block_size
|
||||
)
|
||||
scheduler._num_workers = 1
|
||||
scheduler._next_store_job_id = 0
|
||||
scheduler._pinned_saves = {}
|
||||
return scheduler
|
||||
|
||||
|
||||
@@ -64,6 +72,14 @@ def _make_preemption_scheduler_output():
|
||||
)
|
||||
|
||||
|
||||
def _make_worker_output(completed_saves: dict[int, int]) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
kv_connector_worker_meta=MooncakeStoreWorkerMetadata(
|
||||
completed_saves=completed_saves
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _add_unfinished_request(
|
||||
scheduler: MooncakeStoreScheduler,
|
||||
*,
|
||||
@@ -133,7 +149,7 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
|
||||
assert tracker.num_saved_tokens == 48
|
||||
|
||||
|
||||
def test_preemption_resets_tracker_before_request_finished():
|
||||
def test_preemption_resets_tracker():
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
@@ -152,8 +168,6 @@ def test_preemption_resets_tracker_before_request_finished():
|
||||
assert tracker.token_ids is None
|
||||
assert tracker.has_pending_offload is False
|
||||
assert tracker.prefill_end_tokens == 0
|
||||
request = SimpleNamespace(request_id="req-0")
|
||||
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
|
||||
|
||||
|
||||
def test_preemption_clears_stale_load_state():
|
||||
@@ -215,7 +229,7 @@ def test_pending_load_does_not_co_queue_save():
|
||||
# enqueue a save in the same scheduling step. Co-queuing both produces a
|
||||
# recv+send pair for the same req_id, and the scheduler's
|
||||
# _update_from_kv_xfer_finished then trips `assert req_id in self.requests`
|
||||
# when both completions land for the delay-freed request.
|
||||
# when a completion lands for a request it has already dropped.
|
||||
scheduler = _make_bare_scheduler()
|
||||
_make_pending_load_unfinished_request(
|
||||
scheduler,
|
||||
@@ -238,9 +252,7 @@ def test_pending_load_does_not_co_queue_save():
|
||||
# Load is still issued as planned.
|
||||
assert req_meta.load_spec is not None
|
||||
assert req_meta.load_spec.can_load is True
|
||||
# And the tracker's saved-tokens watermark stays at 0 so request_finished
|
||||
# later sees `num_saved_tokens <= 0` and frees immediately rather than
|
||||
# waiting for a finished_sending that will never come.
|
||||
# And the save watermark does not advance for a save that was never queued.
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
|
||||
@@ -661,30 +673,34 @@ def test_disabled_lookup_reports_no_hit_without_querying_client():
|
||||
assert scheduler.load_specs == {}
|
||||
|
||||
|
||||
def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
# A sub-block prompt never produces a block-aligned save, so the partial-
|
||||
# tail offload arriving this step is emitted as an offload-only ReqMeta
|
||||
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
|
||||
# the worker skips the normal save). Pending-offload state delays the free
|
||||
# without advancing the normal-save watermark before the put succeeds.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
def _add_pending_partial_tail_request(
|
||||
scheduler: MooncakeStoreScheduler,
|
||||
*,
|
||||
num_tokens: int,
|
||||
block_hashes: list[bytes],
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> SimpleNamespace:
|
||||
"""Register a sub-block request and return the step that offloads its tail.
|
||||
|
||||
The CoW block holding the tail is block 7, which the core deliberately keeps
|
||||
out of the request's block table.
|
||||
"""
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(12)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
all_token_ids=list(range(num_tokens)),
|
||||
block_hashes=block_hashes,
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._unfinished_requests["req-0"] = (request, block_ids)
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=12,
|
||||
allocated_block_ids=([0],),
|
||||
token_len=num_tokens,
|
||||
allocated_block_ids=block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=list(range(12)),
|
||||
prefill_end_tokens=12,
|
||||
token_ids=list(range(num_tokens)),
|
||||
prefill_end_tokens=num_tokens,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
@@ -699,6 +715,21 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
)
|
||||
|
||||
|
||||
def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
# A sub-block prompt never produces a block-aligned save, so the partial-
|
||||
# tail offload arriving this step is emitted as an offload-only ReqMeta
|
||||
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
|
||||
# the worker skips the normal save), without advancing the normal-save
|
||||
# watermark before the put succeeds.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
out = _add_pending_partial_tail_request(
|
||||
scheduler,
|
||||
num_tokens=12,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
block_ids=([0],),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
@@ -712,42 +743,16 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.has_pending_offload is True
|
||||
request = SimpleNamespace(request_id="req-0")
|
||||
assert scheduler.request_finished(request, ([0],)) == (True, None)
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(20)),
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
out = _add_pending_partial_tail_request(
|
||||
scheduler,
|
||||
num_tokens=20,
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=20,
|
||||
allocated_block_ids=([0, 1],),
|
||||
num_saved_tokens=0,
|
||||
token_ids=list(range(20)),
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
prefill_end_tokens=20,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
block_ids=([0, 1],),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
@@ -791,3 +796,69 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 48
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_partial_tail_cow_block_is_referenced_for_the_job():
|
||||
# The CoW block a partial-tail offload reads is deliberately kept out of the
|
||||
# request block table, so it is absent from ReqMeta.block_ids. The worker
|
||||
# DMAs out of it just as asynchronously, so it needs its own reference.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
out = _add_pending_partial_tail_request(
|
||||
scheduler,
|
||||
num_tokens=12,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
block_ids=([0],),
|
||||
)
|
||||
pool = scheduler._gpu_block_pool
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
|
||||
# puts it last in eviction priority.
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
|
||||
assert pool.blocks[7].ref_cnt == 1
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert pool.blocks[7].ref_cnt == 0
|
||||
|
||||
|
||||
def test_store_job_blocks_are_released_once_every_rank_reports():
|
||||
# Every rank DMAs the job's blocks on its own, so the reference can only be
|
||||
# dropped once the last of them reports. Until then the engine has to keep
|
||||
# stepping: a completion only reaches the scheduler as worker metadata
|
||||
# attached to a step, and a finishing request no longer defers its own free.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler._num_workers = 2
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
pool = scheduler._gpu_block_pool
|
||||
assert scheduler.has_pending_push_work() is False
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_scheduler_output(scheduled_spec_tokens=None)
|
||||
)
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
assert pool.blocks[2].ref_cnt == 1
|
||||
assert scheduler.has_pending_push_work() is True
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert pool.blocks[2].ref_cnt == 1
|
||||
assert scheduler.has_pending_push_work() is True
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert pool.blocks[2].ref_cnt == 0
|
||||
assert scheduler.has_pending_push_work() is False
|
||||
|
||||
|
||||
def test_worker_metadata_aggregates_completions_across_ranks():
|
||||
# The engine merges each rank's metadata before the scheduler sees it, so a
|
||||
# job that every rank finished in one step arrives as a single count.
|
||||
merged = MooncakeStoreWorkerMetadata(completed_saves={1: 1}).aggregate(
|
||||
MooncakeStoreWorkerMetadata(completed_saves={1: 1, 2: 1})
|
||||
)
|
||||
assert merged.completed_saves == {1: 2, 2: 1}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
@@ -156,6 +158,17 @@ def _make_load_req(
|
||||
)
|
||||
|
||||
|
||||
_TEST_SAVE_SEQ = itertools.count(1)
|
||||
|
||||
|
||||
def _run_store_req(thread, req_meta: ReqMeta) -> None:
|
||||
"""Register, enqueue and run a store job the way the worker does."""
|
||||
if req_meta.store_job_id is None:
|
||||
req_meta.store_job_id = next(_TEST_SAVE_SEQ)
|
||||
thread.add_request(req_meta)
|
||||
thread._handle_request(thread.request_queue.get())
|
||||
|
||||
|
||||
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
@@ -385,27 +398,23 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
|
||||
]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread._store_pressure_active is True
|
||||
assert "req-a" in thread._skip_store_requests
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
thread.add_stored_request("req-b")
|
||||
thread._handle_request(_make_store_req("req-b", [b"b0", b"b1"]))
|
||||
_run_store_req(thread, _make_store_req("req-b", [b"b0", b"b1"]))
|
||||
|
||||
assert thread._store_pressure_active is False
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a4", b"a5"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a4", b"a5"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 3
|
||||
|
||||
@@ -418,8 +427,7 @@ def test_store_sending_thread_records_mooncake_metrics():
|
||||
stats = MooncakeStoreConnectorStats()
|
||||
thread._record_operation_cb = stats.record_operation
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert len(stats.data["save_exists"]) == 1
|
||||
assert stats.data["save_exists"][0]["num_keys"] == 2
|
||||
@@ -497,16 +505,16 @@ def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -523,16 +531,16 @@ def test_store_sending_thread_delta_strides_with_local_phase():
|
||||
store.batch_put_from_multi_buffers.return_value = [256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -550,15 +558,15 @@ def test_tp_sharded_group_saves_every_block_on_every_rank():
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
thread.group_put_steps = [1]
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -576,15 +584,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=16,
|
||||
block_ids=([0],),
|
||||
block_hashes=[b"a0"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
@@ -595,15 +603,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
|
||||
# The next batch resumes from offset 0, re-covering the chunk skipped under
|
||||
# pressure (chunk 0) rather than losing it.
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -703,13 +711,11 @@ def test_partial_tail_offload_honors_active_pressure_gate():
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
thread.add_stored_request("req-a")
|
||||
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
|
||||
def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
@@ -717,17 +723,14 @@ def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
thread.add_stored_request("req-a")
|
||||
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
assert thread._store_pressure_active is True
|
||||
assert thread._skip_store_requests == {"req-a"}
|
||||
assert thread._saved_offset.get("req-a", 0) == 0
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
|
||||
@@ -737,16 +740,16 @@ def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -790,16 +793,16 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
|
||||
token_databases=[db_full, db_masked],
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -845,15 +848,15 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
|
||||
coord=coord,
|
||||
token_databases=[db0, db1],
|
||||
)
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=48,
|
||||
block_ids=([0, 1, 2], [2, 1, 0]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
db0.prepare_value.assert_not_called()
|
||||
@@ -881,46 +884,71 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread._store_pressure_active is False
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
|
||||
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
|
||||
# `batch_is_exist` raising must still decrement `stored_requests` so the
|
||||
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
with pytest.raises(RuntimeError):
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
|
||||
|
||||
def test_store_sending_thread_releases_pin_on_batch_put_failure():
|
||||
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
|
||||
# pin must still be released through the finally block.
|
||||
@pytest.mark.parametrize(
|
||||
"failing_call", ["batch_is_exist", "batch_put_from_multi_buffers"]
|
||||
)
|
||||
def test_store_sending_thread_reports_job_when_store_raises(failing_call):
|
||||
# A store that blows up must still report its job, or the scheduler keeps
|
||||
# the job's GPU block references for the rest of the run.
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
|
||||
getattr(store, failing_call).side_effect = RuntimeError("mooncake down")
|
||||
thread = _make_store_sending_thread(store)
|
||||
req = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
with contextlib.suppress(RuntimeError):
|
||||
_run_store_req(thread, req)
|
||||
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
assert thread.take_completed_saves() == {req.store_job_id: 1}
|
||||
|
||||
|
||||
def test_store_sending_thread_reports_job_when_the_preamble_raises():
|
||||
# The report has to survive a failure before the first store call too, so
|
||||
# every dequeue leaves through the same exit.
|
||||
thread = _make_store_sending_thread(MagicMock())
|
||||
req = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
req.token_len_chunk = None # type: ignore[assignment]
|
||||
|
||||
with contextlib.suppress(TypeError):
|
||||
_run_store_req(thread, req)
|
||||
|
||||
assert thread.take_completed_saves() == {req.store_job_id: 1}
|
||||
thread.request_queue.task_done.assert_called_once()
|
||||
|
||||
|
||||
def test_stale_store_job_cannot_touch_a_reused_request_id():
|
||||
# A preempted request resumes under its original id, so a job left over from
|
||||
# the retired generation carries a req_id that now belongs to a live one.
|
||||
thread = _make_store_sending_thread(MagicMock())
|
||||
stale = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
stale.store_job_id = 1
|
||||
thread.add_request(stale)
|
||||
thread._record_saved(stale, 32)
|
||||
thread.delete_finished_stored_request("req-a")
|
||||
|
||||
live = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
live.store_job_id = 2
|
||||
thread.add_request(live)
|
||||
|
||||
thread.finish_store_job(stale)
|
||||
thread._record_saved(stale, 64)
|
||||
thread._mark_request_skipped_for_pressure(stale)
|
||||
|
||||
assert thread.is_live_store_job(live)
|
||||
assert not thread.is_live_store_job(stale)
|
||||
assert thread._saved_offset.get("req-a") is None
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids():
|
||||
@@ -991,8 +1019,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set
|
||||
replicate_config = SimpleNamespace(preferred_segment="10.0.0.7:50053")
|
||||
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
call_args = store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -1010,8 +1037,7 @@ def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_
|
||||
replicate_config = SimpleNamespace()
|
||||
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
call_args = store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -1067,8 +1093,7 @@ def test_store_sending_thread_sets_group_ids_when_enabled():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -1092,8 +1117,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_flag_disabled():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
|
||||
@@ -1112,8 +1136,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_unsupported():
|
||||
supports_group_ids=False,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
|
||||
@@ -1180,8 +1203,7 @@ def test_store_sending_thread_group_id_excludes_physical_sharding():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
@@ -1210,8 +1232,7 @@ def test_store_sending_thread_multiple_segments_share_logical_group_id():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
@@ -1260,8 +1281,7 @@ def test_store_sending_thread_group_ids_share_across_kv_cache_groups():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_multi_group_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_multi_group_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
@@ -1296,8 +1316,7 @@ def test_store_sending_thread_group_ids_follow_missing_key_filter():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == ["test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131"]
|
||||
@@ -1866,15 +1885,15 @@ def test_store_sending_thread_clamps_token_len_to_lcm():
|
||||
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=33,
|
||||
block_ids=([0, 1, 2],),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
@@ -1905,20 +1924,19 @@ def test_store_sending_thread_skips_when_token_len_below_lcm():
|
||||
store, coord=coord, token_databases=[db], block_size=64
|
||||
)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=32,
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=[b"a0", b"a1"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["r0"] == 0
|
||||
|
||||
|
||||
def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
@@ -1982,15 +2000,15 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
@@ -2057,16 +2075,16 @@ def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._saved_offset["r0"] = 32
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
@@ -2128,8 +2146,8 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
thread.enable_kv_event = True
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(4)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=32,
|
||||
@@ -2137,7 +2155,7 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
token_ids=list(range(32)),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
full_event, swa_event = thread.get_kv_events()
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorBase_V1,
|
||||
KVConnectorMetadata,
|
||||
KVConnectorRole,
|
||||
KVConnectorWorkerMetadata,
|
||||
SupportsHMA,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
@@ -37,6 +38,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.metrics import (
|
||||
from vllm.forward_context import ForwardContext
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.attention.backend import AttentionMetadata
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.sched.output import SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
@@ -201,6 +203,14 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request, blocks, num_external_tokens
|
||||
)
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.bind_gpu_block_pool(gpu_block_pool)
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.has_pending_push_work()
|
||||
|
||||
def build_connector_meta(
|
||||
self,
|
||||
scheduler_output: SchedulerOutput,
|
||||
@@ -208,6 +218,10 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.build_connector_meta(scheduler_output)
|
||||
|
||||
def build_connector_worker_meta(self) -> KVConnectorWorkerMetadata | None:
|
||||
assert self.connector_worker is not None
|
||||
return self.connector_worker.build_connector_worker_meta()
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
@@ -220,8 +234,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
assert self.connector_scheduler is not None
|
||||
return self.connector_scheduler.request_finished(request, block_ids)
|
||||
# An in-flight store job holds its own reference on the blocks it reads,
|
||||
# so a finishing request never has to defer freeing them.
|
||||
return False, None
|
||||
|
||||
def reset_cache(self) -> bool | None:
|
||||
"""Reset the external Mooncake store on prefix-cache reset.
|
||||
@@ -241,6 +256,9 @@ class MooncakeStoreConnector(KVConnectorBase_V1, SupportsHMA):
|
||||
return None
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput):
|
||||
assert self.connector_scheduler is not None
|
||||
self.connector_scheduler.update_connector_output(connector_output)
|
||||
|
||||
kv_cache_events = connector_output.kv_cache_events
|
||||
if not kv_cache_events or not isinstance(
|
||||
kv_cache_events, MooncakeStoreKVEvents
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"""Data classes for MooncakeStoreConnector."""
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass
|
||||
from dataclasses import dataclass, field
|
||||
from typing import cast
|
||||
|
||||
import numpy as np
|
||||
@@ -14,6 +14,7 @@ import torch
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
KVConnectorWorkerMetadata,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.math_utils import cdiv
|
||||
@@ -366,6 +367,10 @@ class ReqMeta:
|
||||
|
||||
token_ids: list[int] | None = None
|
||||
num_prompt_tokens: int | None = None
|
||||
# Identifies this store job for the engine's lifetime. A request id cannot
|
||||
# serve that purpose: it is reused once a preempted request resumes, so it
|
||||
# would release the wrong job's blocks.
|
||||
store_job_id: int | None = None
|
||||
# Core-provided per-mamba-group
|
||||
# (group_id, cow_block_id, boundary_tokens) for this request's partial tail.
|
||||
# Present only on the producer's CoW step; drives the connector's offload
|
||||
@@ -434,6 +439,23 @@ class ReqMeta:
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MooncakeStoreWorkerMetadata(KVConnectorWorkerMetadata):
|
||||
"""Maps ``ReqMeta.store_job_id`` to the number of ranks done with that job."""
|
||||
|
||||
completed_saves: dict[int, int] = field(default_factory=dict)
|
||||
|
||||
def aggregate(
|
||||
self, other: "KVConnectorWorkerMetadata"
|
||||
) -> "MooncakeStoreWorkerMetadata":
|
||||
assert isinstance(other, MooncakeStoreWorkerMetadata)
|
||||
for store_job_id, count in other.completed_saves.items():
|
||||
self.completed_saves[store_job_id] = (
|
||||
self.completed_saves.get(store_job_id, 0) + count
|
||||
)
|
||||
return self
|
||||
|
||||
|
||||
class MooncakeStoreConnectorMetadata(KVConnectorMetadata):
|
||||
"""Metadata passed from scheduler to worker."""
|
||||
|
||||
|
||||
@@ -5,8 +5,6 @@
|
||||
# (vllm_ascend/distributed/kv_transfer/kv_pool/ascend_store/).
|
||||
"""Scheduler-side logic for MooncakeStoreConnector."""
|
||||
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.base import (
|
||||
KVConnectorMetadata,
|
||||
@@ -17,6 +15,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.coordinator imp
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import ( # noqa: E501
|
||||
LoadSpec,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
)
|
||||
@@ -24,10 +23,12 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.worker import (
|
||||
LookupKeyClient,
|
||||
)
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
from vllm.v1.core.kv_cache_manager import KVCacheBlocks
|
||||
from vllm.v1.core.kv_cache_utils import resolve_kv_cache_block_sizes
|
||||
from vllm.v1.core.sched.output import NewRequestData, SchedulerOutput
|
||||
from vllm.v1.kv_cache_interface import KVCacheConfig
|
||||
from vllm.v1.outputs import KVConnectorOutput
|
||||
from vllm.v1.request import Request
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -78,6 +79,15 @@ class MooncakeStoreScheduler:
|
||||
self._unfinished_requests: dict[str, tuple[Request, tuple[list[int], ...]]] = {}
|
||||
self._unfinished_request_ids: set[str] = set()
|
||||
|
||||
self._gpu_block_pool: BlockPool | None = None
|
||||
self._num_workers = vllm_config.parallel_config.world_size
|
||||
self._next_store_job_id = 0
|
||||
# store_job_id -> (referenced block ids, ranks yet to report completion)
|
||||
self._pinned_saves: dict[int, tuple[list[int], int]] = {}
|
||||
|
||||
def bind_gpu_block_pool(self, gpu_block_pool: BlockPool) -> None:
|
||||
self._gpu_block_pool = gpu_block_pool
|
||||
|
||||
def get_num_new_matched_tokens(
|
||||
self,
|
||||
request: Request,
|
||||
@@ -392,33 +402,75 @@ class MooncakeStoreScheduler:
|
||||
)
|
||||
)
|
||||
|
||||
self._reference_save_blocks(meta)
|
||||
return meta
|
||||
|
||||
def request_finished(
|
||||
self,
|
||||
request: Request,
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> tuple[bool, dict[str, Any] | None]:
|
||||
"""Determine whether to delay freeing blocks for async save."""
|
||||
if self.kv_role == "kv_consumer":
|
||||
return False, None
|
||||
tracker = self._request_trackers.get(request.request_id)
|
||||
# Missing tracker can happen when the request is aborted before the
|
||||
# connector observes the normal finished lifecycle or is preempted
|
||||
# before finishing.
|
||||
if tracker is None or (
|
||||
tracker.num_saved_tokens <= 0 and not tracker.has_pending_offload
|
||||
):
|
||||
return False, None
|
||||
total_blocks = sum(len(g) for g in block_ids)
|
||||
delay_free_blocks = total_blocks > 0
|
||||
if delay_free_blocks:
|
||||
logger.debug(
|
||||
"Delaying free of %d blocks for request %s",
|
||||
total_blocks,
|
||||
request.request_id,
|
||||
def _reference_save_blocks(self, meta: MooncakeStoreConnectorMetadata) -> None:
|
||||
"""Take a GPU block reference for every store job this step emits.
|
||||
|
||||
The worker DMAs out of these blocks after the step that scheduled them,
|
||||
so a reference keeps them out of the free queue even once the request
|
||||
itself is freed, until every rank reports the job done.
|
||||
"""
|
||||
pool = self._gpu_block_pool
|
||||
for req_meta in meta.requests:
|
||||
if not req_meta.can_save:
|
||||
continue
|
||||
assert pool is not None, (
|
||||
"GPU block pool must be bound before any store job is emitted"
|
||||
)
|
||||
return delay_free_blocks, None
|
||||
req_meta.store_job_id = store_job_id = self._next_store_job_id
|
||||
self._next_store_job_id += 1
|
||||
block_ids: list[int] = []
|
||||
if req_meta.partial_tail_offloads:
|
||||
# A partial-tail CoW block is deliberately kept out of the
|
||||
# request's block table, so it is absent from `block_ids` even
|
||||
# though the worker DMAs out of it just as asynchronously.
|
||||
# It leads the list, as in `pop_blocks_for_free`.
|
||||
block_ids += [bid for _, bid, _ in req_meta.partial_tail_offloads]
|
||||
# Every allocated block is referenced, not just the ones covering
|
||||
# this job's token range: a rank resumes from its own last
|
||||
# successful offset, which lags the scheduler's whenever a save was
|
||||
# skipped or failed, so it may read anywhere below the range.
|
||||
block_ids += [bid for group in req_meta.block_ids for bid in group]
|
||||
if not block_ids:
|
||||
continue
|
||||
self._pinned_saves[store_job_id] = (block_ids, self._num_workers)
|
||||
pool.touch([pool.blocks[bid] for bid in block_ids])
|
||||
|
||||
def update_connector_output(self, connector_output: KVConnectorOutput) -> None:
|
||||
"""Drop the block references of store jobs every rank has finished."""
|
||||
meta = connector_output.kv_connector_worker_meta
|
||||
if not isinstance(meta, MooncakeStoreWorkerMetadata):
|
||||
return
|
||||
pool = self._gpu_block_pool
|
||||
assert pool is not None
|
||||
for store_job_id, count in meta.completed_saves.items():
|
||||
pinned = self._pinned_saves.get(store_job_id)
|
||||
if pinned is None:
|
||||
# The job referenced no blocks, so nothing was recorded for it.
|
||||
continue
|
||||
block_ids, remaining = pinned
|
||||
remaining -= count
|
||||
if remaining > 0:
|
||||
self._pinned_saves[store_job_id] = (block_ids, remaining)
|
||||
continue
|
||||
assert remaining == 0, (
|
||||
f"store job {store_job_id} reported by too many ranks"
|
||||
)
|
||||
del self._pinned_saves[store_job_id]
|
||||
# Tail-first, as elsewhere, so the shared prefix is evicted last.
|
||||
pool.free_blocks(pool.blocks[bid] for bid in reversed(block_ids))
|
||||
|
||||
def has_pending_push_work(self) -> bool:
|
||||
"""Keep the engine stepping while any store job still holds block refs.
|
||||
|
||||
Completions only reach the scheduler as worker metadata on a step, so an
|
||||
engine that quiesced with jobs in flight would leave those references
|
||||
held indefinitely. Nothing else keeps it alive now that a finishing
|
||||
request no longer defers its own free.
|
||||
"""
|
||||
return bool(self._pinned_saves)
|
||||
|
||||
def reset_store(self) -> bool:
|
||||
"""Trigger a global ``remove_all(force=True)`` on the Mooncake master.
|
||||
|
||||
@@ -17,7 +17,6 @@ import queue
|
||||
import socket
|
||||
import threading
|
||||
import time
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Sequence
|
||||
from concurrent.futures import Future, ThreadPoolExecutor
|
||||
from dataclasses import dataclass
|
||||
@@ -46,6 +45,7 @@ from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
ChunkedTokenDatabase,
|
||||
KeyMetadata,
|
||||
MooncakeStoreConnectorMetadata,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
PoolKey,
|
||||
ReqMeta,
|
||||
)
|
||||
@@ -506,7 +506,16 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self.group_put_steps = group_put_steps
|
||||
self.coord = coord
|
||||
self.kv_role = kv_role
|
||||
self.stored_requests: defaultdict[str, int] = defaultdict(int)
|
||||
# req_id -> ids of its store jobs that are still queued or running.
|
||||
# Keying by store_job_id, which never repeats for the engine's lifetime,
|
||||
# rather than counting jobs per request id makes the ledger immune to id
|
||||
# reuse across preemption: a job left over from a retired generation is
|
||||
# missing from the set its resumed generation builds, so it can no longer
|
||||
# retire that generation, rewind its resume offset, or mark it skipped.
|
||||
self.stored_requests: dict[str, set[int]] = {}
|
||||
# store_job_id -> times this rank finished with it, drained every step
|
||||
# so the scheduler can release the blocks it referenced for those jobs.
|
||||
self._completed_saves: dict[int, int] = {}
|
||||
self.enable_kv_event = enable_kv_event
|
||||
# Caller always passes a non-None ReplicateConfig — see
|
||||
# MooncakeStoreWorker.__init__ where store_replicate_config is built.
|
||||
@@ -522,14 +531,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
# batch resumes here, so pressure-skipped or failed ranges are retried.
|
||||
self._saved_offset: dict[str, int] = {}
|
||||
|
||||
def add_stored_request(self, req_id: str):
|
||||
def add_request(self, request: ReqMeta) -> None:
|
||||
# Register before enqueueing so a job is never picked up unledgered.
|
||||
assert request.store_job_id is not None
|
||||
with self.done_task_lock:
|
||||
self.stored_requests[req_id] += 1
|
||||
self.stored_requests.setdefault(request.req_id, set()).add(
|
||||
request.store_job_id
|
||||
)
|
||||
super().add_request(request)
|
||||
|
||||
def dec_stored_request(self, req_id: str):
|
||||
def is_live_store_job(self, req_meta: ReqMeta) -> bool:
|
||||
with self.done_task_lock:
|
||||
if req_id in self.stored_requests:
|
||||
self.stored_requests[req_id] -= 1
|
||||
return req_meta.store_job_id in self.stored_requests.get(
|
||||
req_meta.req_id, ()
|
||||
)
|
||||
|
||||
def delete_finished_stored_request(self, req_id: str):
|
||||
with self.done_task_lock:
|
||||
@@ -538,21 +553,51 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
self._skip_store_requests.discard(req_id)
|
||||
self._saved_offset.pop(req_id, None)
|
||||
|
||||
def _record_saved(self, req_id: str, token_len: int) -> None:
|
||||
# Guard on liveness so a concurrent finish/preempt pop isn't recreated.
|
||||
def finish_store_job(self, req_meta: ReqMeta) -> None:
|
||||
"""Retire a job from the ledger and report its blocks as no longer read.
|
||||
|
||||
Every path out of a job must reach this, skips and failures included: a
|
||||
job that never reports leaves its blocks referenced for the rest of the
|
||||
run. The discard is a no-op for a job whose generation already retired.
|
||||
"""
|
||||
store_job_id = req_meta.store_job_id
|
||||
assert store_job_id is not None, (
|
||||
"a queued store job always carries a store_job_id"
|
||||
)
|
||||
with self.done_task_lock:
|
||||
if req_id in self.stored_requests:
|
||||
self._saved_offset[req_id] = token_len
|
||||
live = self.stored_requests.get(req_meta.req_id)
|
||||
if live is not None:
|
||||
live.discard(store_job_id)
|
||||
self._completed_saves[store_job_id] = (
|
||||
self._completed_saves.get(store_job_id, 0) + 1
|
||||
)
|
||||
|
||||
def take_completed_saves(self) -> dict[int, int]:
|
||||
with self.done_task_lock:
|
||||
completed = self._completed_saves
|
||||
self._completed_saves = {}
|
||||
return completed
|
||||
|
||||
def _record_saved(self, req_meta: ReqMeta, token_len: int) -> None:
|
||||
# Guard on job liveness so neither a concurrent finish/preempt pop nor a
|
||||
# stale job's offset is written back over the live generation's.
|
||||
with self.done_task_lock:
|
||||
if req_meta.store_job_id in self.stored_requests.get(req_meta.req_id, ()):
|
||||
self._saved_offset[req_meta.req_id] = token_len
|
||||
|
||||
def _should_skip_request(self, req_id: str) -> bool:
|
||||
with self.done_task_lock:
|
||||
return self._store_pressure_active and req_id in self._skip_store_requests
|
||||
|
||||
def _mark_request_skipped_for_pressure(self, req_id: str) -> bool:
|
||||
def _mark_request_skipped_for_pressure(self, req_meta: ReqMeta) -> bool:
|
||||
req_id = req_meta.req_id
|
||||
with self.done_task_lock:
|
||||
already_skipped = req_id in self._skip_store_requests
|
||||
self._store_pressure_active = True
|
||||
self._skip_store_requests.add(req_id)
|
||||
# The pressure itself is global, but only a live job may sentence its
|
||||
# own request to being skipped.
|
||||
if req_meta.store_job_id in self.stored_requests.get(req_id, ()):
|
||||
self._skip_store_requests.add(req_id)
|
||||
return already_skipped
|
||||
|
||||
def _clear_store_pressure(self) -> bool:
|
||||
@@ -727,7 +772,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
failed_codes,
|
||||
)
|
||||
if MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes:
|
||||
self._mark_request_skipped_for_pressure(req_meta.req_id)
|
||||
self._mark_request_skipped_for_pressure(req_meta)
|
||||
return False
|
||||
|
||||
if self._clear_store_pressure():
|
||||
@@ -738,22 +783,20 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
return True
|
||||
|
||||
def _handle_request(self, req_meta: ReqMeta):
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens, which
|
||||
# is also ``store_mask``'s precondition.
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if req_id not in self.stored_requests:
|
||||
self.request_queue.task_done()
|
||||
return
|
||||
|
||||
# Decrement the in-flight counter and signal task_done() in `finally`
|
||||
# so the scheduler can release the GPU blocks it pinned for this
|
||||
# request (via `delay_free_blocks`) even when the store path raises.
|
||||
# The single `finally` is the only way out, so the scheduler releases
|
||||
# this job's GPU block references however the job ends.
|
||||
try:
|
||||
# Cache hits are always a multiple of ``lcm_block_size`` tokens,
|
||||
# which is also ``store_mask``'s precondition.
|
||||
lcm_block_size = self.coord.lcm_block_size
|
||||
token_len = req_meta.token_len_chunk // lcm_block_size * lcm_block_size
|
||||
block_ids_per_group = req_meta.block_ids
|
||||
req_id = req_meta.req_id
|
||||
current_event = req_meta.current_event
|
||||
|
||||
if not self.is_live_store_job(req_meta):
|
||||
return
|
||||
|
||||
if self._should_skip_request(req_id):
|
||||
logger.debug(
|
||||
"Skipping Mooncake store for request %s while CPU/disk "
|
||||
@@ -808,7 +851,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
group_indices.append(g_idx)
|
||||
|
||||
if not keys:
|
||||
self._record_saved(req_id, token_len)
|
||||
self._record_saved(req_meta, token_len)
|
||||
return
|
||||
|
||||
# Check which blocks already exist (dedup)
|
||||
@@ -834,7 +877,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
]
|
||||
|
||||
if not missing_indices:
|
||||
self._record_saved(req_id, token_len)
|
||||
self._record_saved(req_meta, token_len)
|
||||
return
|
||||
|
||||
if len(missing_indices) != len(keys):
|
||||
@@ -954,7 +997,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
)
|
||||
if (
|
||||
MOONCAKE_NO_AVAILABLE_HANDLE in failed_codes
|
||||
and not self._mark_request_skipped_for_pressure(req_id)
|
||||
and not self._mark_request_skipped_for_pressure(req_meta)
|
||||
):
|
||||
logger.warning(
|
||||
"Detected Mooncake CPU/disk offloading pressure "
|
||||
@@ -964,7 +1007,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
req_id,
|
||||
)
|
||||
else:
|
||||
self._record_saved(req_id, token_len)
|
||||
self._record_saved(req_meta, token_len)
|
||||
if self._clear_store_pressure():
|
||||
logger.info(
|
||||
"Mooncake CPU/disk offloading pressure cleared "
|
||||
@@ -984,7 +1027,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
||||
if self.enable_kv_event and stored_events:
|
||||
self.update_kv_event(stored_events)
|
||||
finally:
|
||||
self.dec_stored_request(req_id)
|
||||
self.finish_store_job(req_meta)
|
||||
self.request_queue.task_done()
|
||||
|
||||
|
||||
@@ -1670,16 +1713,15 @@ class MooncakeStoreWorker:
|
||||
continue
|
||||
request.current_event = current_event
|
||||
assert self.kv_send_thread is not None
|
||||
self.kv_send_thread.add_stored_request(request.req_id)
|
||||
self.kv_send_thread.add_request(request)
|
||||
|
||||
# Check completion of previously queued transfers
|
||||
done_sending = (
|
||||
self._get_and_clear_finished_sending(finished_req_ids, meta)
|
||||
if self.kv_role in ["kv_producer", "kv_both"]
|
||||
else set()
|
||||
)
|
||||
if self.kv_role in ["kv_producer", "kv_both"]:
|
||||
self._close_ended_store_requests(finished_req_ids, meta)
|
||||
|
||||
# Blocks read by a store job are released by the scheduler when the job
|
||||
# reports back (see build_connector_worker_meta), so no request ever waits
|
||||
# on a `finished_sending` signal to get its blocks back.
|
||||
done_sending: set[str] = set()
|
||||
done_recving: set[str] = set()
|
||||
if self.load_async:
|
||||
for recv_thread in self.kv_recv_threads:
|
||||
@@ -1727,35 +1769,37 @@ class MooncakeStoreWorker:
|
||||
self.kv_connector_stats = MooncakeStoreConnectorStats()
|
||||
return kv_connector_stats
|
||||
|
||||
def _get_and_clear_finished_sending(
|
||||
def _close_ended_store_requests(
|
||||
self,
|
||||
finished_req_ids: set[str],
|
||||
meta: MooncakeStoreConnectorMetadata,
|
||||
) -> set[str]:
|
||||
) -> None:
|
||||
"""Retire the ledger entries of requests that finished or were preempted.
|
||||
|
||||
An entry may only go once its jobs have drained, because they still read
|
||||
the resume offset it owns; a request that comes back after preemption
|
||||
then saves from the start rather than from where the last attempt got to.
|
||||
"""
|
||||
assert self.kv_send_thread is not None
|
||||
finished_sending: set[str] = set()
|
||||
|
||||
for req_id in meta.preempted_req_ids:
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
|
||||
for req_id in self.kv_send_thread.stored_requests.copy():
|
||||
if (
|
||||
self.kv_send_thread.stored_requests[req_id] == 0
|
||||
and req_id in self.finished_store_req
|
||||
):
|
||||
self.finished_store_req.remove(req_id)
|
||||
finished_sending.add(req_id)
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
|
||||
for req_id in finished_req_ids:
|
||||
req_remain_jobs = self.kv_send_thread.stored_requests.get(req_id)
|
||||
if req_remain_jobs == 0:
|
||||
finished_sending.add(req_id)
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
elif req_remain_jobs is not None:
|
||||
for req_id in finished_req_ids | self.finished_store_req:
|
||||
if self.kv_send_thread.stored_requests.get(req_id):
|
||||
# Queued jobs still need the resume offset; retire on a later step.
|
||||
self.finished_store_req.add(req_id)
|
||||
else:
|
||||
self.finished_store_req.discard(req_id)
|
||||
self.kv_send_thread.delete_finished_stored_request(req_id)
|
||||
|
||||
return finished_sending
|
||||
def build_connector_worker_meta(self) -> MooncakeStoreWorkerMetadata | None:
|
||||
if self.kv_send_thread is None:
|
||||
return None
|
||||
completed_saves = self.kv_send_thread.take_completed_saves()
|
||||
if not completed_saves:
|
||||
return None
|
||||
return MooncakeStoreWorkerMetadata(completed_saves=completed_saves)
|
||||
|
||||
def lookup(self, num_tokens: int, block_hashes: Sequence[BlockHash]) -> int:
|
||||
"""Check how many prefix tokens exist in the store.
|
||||
|
||||
Reference in New Issue
Block a user