[Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372)
Signed-off-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
@@ -228,12 +228,11 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
store_job_id=1,
|
||||
)
|
||||
send_thread.add_stored_request("r0")
|
||||
# Put the request in the queue so task_done() doesn't underflow.
|
||||
send_thread.request_queue.put(save_req)
|
||||
req = send_thread.request_queue.get()
|
||||
send_thread._handle_request(req)
|
||||
# add_request also queues the job, so task_done() doesn't underflow.
|
||||
send_thread.add_request(save_req)
|
||||
send_thread._handle_request(send_thread.request_queue.get())
|
||||
|
||||
# Point worker.store at the dict store (the worker constructor captured
|
||||
# the MagicMock; replace with the real dict store for lookup).
|
||||
@@ -499,15 +498,15 @@ def test_offload_syncs_event_before_put():
|
||||
can_save=True,
|
||||
num_prompt_tokens=12,
|
||||
partial_tail_offloads=[(1, 7, 12)],
|
||||
store_job_id=1,
|
||||
)
|
||||
req.current_event = event
|
||||
send.add_stored_request("r1")
|
||||
|
||||
send.request_queue.put(req)
|
||||
send.add_request(req)
|
||||
send._handle_request(send.request_queue.get())
|
||||
assert send.request_queue.qsize() == 0
|
||||
assert store._data
|
||||
assert send.stored_requests["r1"] == 0
|
||||
assert send.stored_requests["r1"] == set()
|
||||
event.synchronize.assert_called_once()
|
||||
|
||||
|
||||
|
||||
@@ -5,12 +5,14 @@ from types import SimpleNamespace
|
||||
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
|
||||
LoadSpec,
|
||||
MooncakeStoreWorkerMetadata,
|
||||
ReqMeta,
|
||||
RequestTracker,
|
||||
)
|
||||
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
|
||||
MooncakeStoreScheduler,
|
||||
)
|
||||
from vllm.v1.core.block_pool import BlockPool
|
||||
|
||||
|
||||
def _make_bare_scheduler(
|
||||
@@ -27,6 +29,12 @@ def _make_bare_scheduler(
|
||||
scheduler._unfinished_request_ids = {"req-0"}
|
||||
scheduler._unfinished_requests = {}
|
||||
scheduler._request_trackers = {}
|
||||
scheduler._gpu_block_pool = BlockPool(
|
||||
num_gpu_blocks=64, enable_caching=True, hash_block_size=hash_block_size
|
||||
)
|
||||
scheduler._num_workers = 1
|
||||
scheduler._next_store_job_id = 0
|
||||
scheduler._pinned_saves = {}
|
||||
return scheduler
|
||||
|
||||
|
||||
@@ -64,6 +72,14 @@ def _make_preemption_scheduler_output():
|
||||
)
|
||||
|
||||
|
||||
def _make_worker_output(completed_saves: dict[int, int]) -> SimpleNamespace:
|
||||
return SimpleNamespace(
|
||||
kv_connector_worker_meta=MooncakeStoreWorkerMetadata(
|
||||
completed_saves=completed_saves
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _add_unfinished_request(
|
||||
scheduler: MooncakeStoreScheduler,
|
||||
*,
|
||||
@@ -133,7 +149,7 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
|
||||
assert tracker.num_saved_tokens == 48
|
||||
|
||||
|
||||
def test_preemption_resets_tracker_before_request_finished():
|
||||
def test_preemption_resets_tracker():
|
||||
scheduler = _make_bare_scheduler()
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
@@ -152,8 +168,6 @@ def test_preemption_resets_tracker_before_request_finished():
|
||||
assert tracker.token_ids is None
|
||||
assert tracker.has_pending_offload is False
|
||||
assert tracker.prefill_end_tokens == 0
|
||||
request = SimpleNamespace(request_id="req-0")
|
||||
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
|
||||
|
||||
|
||||
def test_preemption_clears_stale_load_state():
|
||||
@@ -215,7 +229,7 @@ def test_pending_load_does_not_co_queue_save():
|
||||
# enqueue a save in the same scheduling step. Co-queuing both produces a
|
||||
# recv+send pair for the same req_id, and the scheduler's
|
||||
# _update_from_kv_xfer_finished then trips `assert req_id in self.requests`
|
||||
# when both completions land for the delay-freed request.
|
||||
# when a completion lands for a request it has already dropped.
|
||||
scheduler = _make_bare_scheduler()
|
||||
_make_pending_load_unfinished_request(
|
||||
scheduler,
|
||||
@@ -238,9 +252,7 @@ def test_pending_load_does_not_co_queue_save():
|
||||
# Load is still issued as planned.
|
||||
assert req_meta.load_spec is not None
|
||||
assert req_meta.load_spec.can_load is True
|
||||
# And the tracker's saved-tokens watermark stays at 0 so request_finished
|
||||
# later sees `num_saved_tokens <= 0` and frees immediately rather than
|
||||
# waiting for a finished_sending that will never come.
|
||||
# And the save watermark does not advance for a save that was never queued.
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
|
||||
@@ -661,30 +673,34 @@ def test_disabled_lookup_reports_no_hit_without_querying_client():
|
||||
assert scheduler.load_specs == {}
|
||||
|
||||
|
||||
def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
# A sub-block prompt never produces a block-aligned save, so the partial-
|
||||
# tail offload arriving this step is emitted as an offload-only ReqMeta
|
||||
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
|
||||
# the worker skips the normal save). Pending-offload state delays the free
|
||||
# without advancing the normal-save watermark before the put succeeds.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
def _add_pending_partial_tail_request(
|
||||
scheduler: MooncakeStoreScheduler,
|
||||
*,
|
||||
num_tokens: int,
|
||||
block_hashes: list[bytes],
|
||||
block_ids: tuple[list[int], ...],
|
||||
) -> SimpleNamespace:
|
||||
"""Register a sub-block request and return the step that offloads its tail.
|
||||
|
||||
The CoW block holding the tail is block 7, which the core deliberately keeps
|
||||
out of the request's block table.
|
||||
"""
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(12)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
all_token_ids=list(range(num_tokens)),
|
||||
block_hashes=block_hashes,
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0],))
|
||||
scheduler._unfinished_requests["req-0"] = (request, block_ids)
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=12,
|
||||
allocated_block_ids=([0],),
|
||||
token_len=num_tokens,
|
||||
allocated_block_ids=block_ids,
|
||||
num_saved_tokens=0,
|
||||
token_ids=list(range(12)),
|
||||
prefill_end_tokens=12,
|
||||
token_ids=list(range(num_tokens)),
|
||||
prefill_end_tokens=num_tokens,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
return SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
@@ -699,6 +715,21 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
)
|
||||
|
||||
|
||||
def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
# A sub-block prompt never produces a block-aligned save, so the partial-
|
||||
# tail offload arriving this step is emitted as an offload-only ReqMeta
|
||||
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
|
||||
# the worker skips the normal save), without advancing the normal-save
|
||||
# watermark before the put succeeds.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
out = _add_pending_partial_tail_request(
|
||||
scheduler,
|
||||
num_tokens=12,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
block_ids=([0],),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
assert len(meta.requests) == 1
|
||||
@@ -712,42 +743,16 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 0
|
||||
assert tracker.has_pending_offload is True
|
||||
request = SimpleNamespace(request_id="req-0")
|
||||
assert scheduler.request_finished(request, ([0],)) == (True, None)
|
||||
|
||||
|
||||
def test_resumed_partial_tail_uses_handoff_boundary():
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
request = SimpleNamespace(
|
||||
all_token_ids=list(range(20)),
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
out = _add_pending_partial_tail_request(
|
||||
scheduler,
|
||||
num_tokens=20,
|
||||
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
|
||||
num_output_placeholders=0,
|
||||
num_prompt_tokens=12,
|
||||
)
|
||||
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
|
||||
scheduler._request_trackers["req-0"] = RequestTracker(
|
||||
req_id="req-0",
|
||||
token_len=20,
|
||||
allocated_block_ids=([0, 1],),
|
||||
num_saved_tokens=0,
|
||||
token_ids=list(range(20)),
|
||||
# Resumption replays prompt + previously generated tokens.
|
||||
prefill_end_tokens=20,
|
||||
)
|
||||
|
||||
out = SimpleNamespace(
|
||||
finished_req_ids=set(),
|
||||
preempted_req_ids=set(),
|
||||
scheduled_new_reqs=[],
|
||||
scheduled_cached_reqs=SimpleNamespace(
|
||||
req_ids=[],
|
||||
new_block_ids=[],
|
||||
num_computed_tokens=[],
|
||||
resumed_req_ids=set(),
|
||||
),
|
||||
num_scheduled_tokens={},
|
||||
scheduled_spec_decode_tokens={},
|
||||
partial_tail_offloads={"req-0": [(1, 7, 12)]},
|
||||
block_ids=([0, 1],),
|
||||
)
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
@@ -791,3 +796,69 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
|
||||
tracker = scheduler._request_trackers["req-0"]
|
||||
assert tracker.num_saved_tokens == 48
|
||||
assert tracker.has_pending_offload is True
|
||||
|
||||
|
||||
def test_partial_tail_cow_block_is_referenced_for_the_job():
|
||||
# The CoW block a partial-tail offload reads is deliberately kept out of the
|
||||
# request block table, so it is absent from ReqMeta.block_ids. The worker
|
||||
# DMAs out of it just as asynchronously, so it needs its own reference.
|
||||
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
|
||||
out = _add_pending_partial_tail_request(
|
||||
scheduler,
|
||||
num_tokens=12,
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
block_ids=([0],),
|
||||
)
|
||||
pool = scheduler._gpu_block_pool
|
||||
|
||||
meta = scheduler.build_connector_meta(out)
|
||||
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
|
||||
# puts it last in eviction priority.
|
||||
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
|
||||
assert pool.blocks[7].ref_cnt == 1
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert pool.blocks[7].ref_cnt == 0
|
||||
|
||||
|
||||
def test_store_job_blocks_are_released_once_every_rank_reports():
|
||||
# Every rank DMAs the job's blocks on its own, so the reference can only be
|
||||
# dropped once the last of them reports. Until then the engine has to keep
|
||||
# stepping: a completion only reaches the scheduler as worker metadata
|
||||
# attached to a step, and a finishing request no longer defers its own free.
|
||||
scheduler = _make_bare_scheduler()
|
||||
scheduler._num_workers = 2
|
||||
_add_unfinished_request(
|
||||
scheduler,
|
||||
token_ids=list(range(48)),
|
||||
block_hashes=[b"h0", b"h1", b"h2"],
|
||||
prefill_end_tokens=48,
|
||||
)
|
||||
pool = scheduler._gpu_block_pool
|
||||
assert scheduler.has_pending_push_work() is False
|
||||
|
||||
meta = scheduler.build_connector_meta(
|
||||
_make_scheduler_output(scheduled_spec_tokens=None)
|
||||
)
|
||||
store_job_id = meta.requests[0].store_job_id
|
||||
assert pool.blocks[2].ref_cnt == 1
|
||||
assert scheduler.has_pending_push_work() is True
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert pool.blocks[2].ref_cnt == 1
|
||||
assert scheduler.has_pending_push_work() is True
|
||||
|
||||
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
|
||||
assert pool.blocks[2].ref_cnt == 0
|
||||
assert scheduler.has_pending_push_work() is False
|
||||
|
||||
|
||||
def test_worker_metadata_aggregates_completions_across_ranks():
|
||||
# The engine merges each rank's metadata before the scheduler sees it, so a
|
||||
# job that every rank finished in one step arrives as a single count.
|
||||
merged = MooncakeStoreWorkerMetadata(completed_saves={1: 1}).aggregate(
|
||||
MooncakeStoreWorkerMetadata(completed_saves={1: 1, 2: 1})
|
||||
)
|
||||
assert merged.completed_saves == {1: 2, 2: 1}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import contextlib
|
||||
import itertools
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
@@ -156,6 +158,17 @@ def _make_load_req(
|
||||
)
|
||||
|
||||
|
||||
_TEST_SAVE_SEQ = itertools.count(1)
|
||||
|
||||
|
||||
def _run_store_req(thread, req_meta: ReqMeta) -> None:
|
||||
"""Register, enqueue and run a store job the way the worker does."""
|
||||
if req_meta.store_job_id is None:
|
||||
req_meta.store_job_id = next(_TEST_SAVE_SEQ)
|
||||
thread.add_request(req_meta)
|
||||
thread._handle_request(thread.request_queue.get())
|
||||
|
||||
|
||||
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
|
||||
return ReqMeta(
|
||||
req_id=req_id,
|
||||
@@ -385,27 +398,23 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
|
||||
]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread._store_pressure_active is True
|
||||
assert "req-a" in thread._skip_store_requests
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
thread.add_stored_request("req-b")
|
||||
thread._handle_request(_make_store_req("req-b", [b"b0", b"b1"]))
|
||||
_run_store_req(thread, _make_store_req("req-b", [b"b0", b"b1"]))
|
||||
|
||||
assert thread._store_pressure_active is False
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a4", b"a5"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a4", b"a5"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 3
|
||||
|
||||
@@ -418,8 +427,7 @@ def test_store_sending_thread_records_mooncake_metrics():
|
||||
stats = MooncakeStoreConnectorStats()
|
||||
thread._record_operation_cb = stats.record_operation
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert len(stats.data["save_exists"]) == 1
|
||||
assert stats.data["save_exists"][0]["num_keys"] == 2
|
||||
@@ -497,16 +505,16 @@ def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -523,16 +531,16 @@ def test_store_sending_thread_delta_strides_with_local_phase():
|
||||
store.batch_put_from_multi_buffers.return_value = [256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -550,15 +558,15 @@ def test_tp_sharded_group_saves_every_block_on_every_rank():
|
||||
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
|
||||
thread.group_put_steps = [1]
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -576,15 +584,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=16,
|
||||
block_ids=([0],),
|
||||
block_hashes=[b"a0"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
@@ -595,15 +603,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
|
||||
|
||||
# The next batch resumes from offset 0, re-covering the chunk skipped under
|
||||
# pressure (chunk 0) rather than losing it.
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -703,13 +711,11 @@ def test_partial_tail_offload_honors_active_pressure_gate():
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
thread._store_pressure_active = True
|
||||
thread._skip_store_requests.add("req-a")
|
||||
thread.add_stored_request("req-a")
|
||||
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
|
||||
def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
@@ -717,17 +723,14 @@ def test_partial_tail_put_failure_activates_pressure_gate():
|
||||
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
|
||||
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
|
||||
thread = _make_partial_tail_send_thread(store)
|
||||
thread.add_stored_request("req-a")
|
||||
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
|
||||
|
||||
assert thread._store_pressure_active is True
|
||||
assert thread._skip_store_requests == {"req-a"}
|
||||
assert thread._saved_offset.get("req-a", 0) == 0
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
|
||||
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
|
||||
@@ -737,16 +740,16 @@ def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
|
||||
store.batch_put_from_multi_buffers.return_value = [256, 256]
|
||||
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 16
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3],),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -790,16 +793,16 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
|
||||
token_databases=[db_full, db_masked],
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._saved_offset["req-a"] = 32
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
|
||||
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_is_exist.call_args.args[0]
|
||||
@@ -845,15 +848,15 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
|
||||
coord=coord,
|
||||
token_databases=[db0, db1],
|
||||
)
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="req-a",
|
||||
token_len_chunk=48,
|
||||
block_ids=([0, 1, 2], [2, 1, 0]),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
db0.prepare_value.assert_not_called()
|
||||
@@ -881,46 +884,71 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
]
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread._store_pressure_active is False
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
|
||||
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
|
||||
# `batch_is_exist` raising must still decrement `stored_requests` so the
|
||||
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
with pytest.raises(RuntimeError):
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
|
||||
|
||||
def test_store_sending_thread_releases_pin_on_batch_put_failure():
|
||||
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
|
||||
# pin must still be released through the finally block.
|
||||
@pytest.mark.parametrize(
|
||||
"failing_call", ["batch_is_exist", "batch_put_from_multi_buffers"]
|
||||
)
|
||||
def test_store_sending_thread_reports_job_when_store_raises(failing_call):
|
||||
# A store that blows up must still report its job, or the scheduler keeps
|
||||
# the job's GPU block references for the rest of the run.
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
|
||||
getattr(store, failing_call).side_effect = RuntimeError("mooncake down")
|
||||
thread = _make_store_sending_thread(store)
|
||||
req = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
with contextlib.suppress(RuntimeError):
|
||||
_run_store_req(thread, req)
|
||||
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
assert thread.take_completed_saves() == {req.store_job_id: 1}
|
||||
|
||||
|
||||
def test_store_sending_thread_reports_job_when_the_preamble_raises():
|
||||
# The report has to survive a failure before the first store call too, so
|
||||
# every dequeue leaves through the same exit.
|
||||
thread = _make_store_sending_thread(MagicMock())
|
||||
req = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
req.token_len_chunk = None # type: ignore[assignment]
|
||||
|
||||
with contextlib.suppress(TypeError):
|
||||
_run_store_req(thread, req)
|
||||
|
||||
assert thread.take_completed_saves() == {req.store_job_id: 1}
|
||||
thread.request_queue.task_done.assert_called_once()
|
||||
|
||||
|
||||
def test_stale_store_job_cannot_touch_a_reused_request_id():
|
||||
# A preempted request resumes under its original id, so a job left over from
|
||||
# the retired generation carries a req_id that now belongs to a live one.
|
||||
thread = _make_store_sending_thread(MagicMock())
|
||||
stale = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
stale.store_job_id = 1
|
||||
thread.add_request(stale)
|
||||
thread._record_saved(stale, 32)
|
||||
thread.delete_finished_stored_request("req-a")
|
||||
|
||||
live = _make_store_req("req-a", [b"a0", b"a1"])
|
||||
live.store_job_id = 2
|
||||
thread.add_request(live)
|
||||
|
||||
thread.finish_store_job(stale)
|
||||
thread._record_saved(stale, 64)
|
||||
thread._mark_request_skipped_for_pressure(stale)
|
||||
|
||||
assert thread.is_live_store_job(live)
|
||||
assert not thread.is_live_store_job(stale)
|
||||
assert thread._saved_offset.get("req-a") is None
|
||||
assert "req-a" not in thread._skip_store_requests
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids():
|
||||
@@ -991,8 +1019,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set
|
||||
replicate_config = SimpleNamespace(preferred_segment="10.0.0.7:50053")
|
||||
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
call_args = store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -1010,8 +1037,7 @@ def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_
|
||||
replicate_config = SimpleNamespace()
|
||||
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
call_args = store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -1067,8 +1093,7 @@ def test_store_sending_thread_sets_group_ids_when_enabled():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
@@ -1092,8 +1117,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_flag_disabled():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
|
||||
@@ -1112,8 +1136,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_unsupported():
|
||||
supports_group_ids=False,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert store.batch_put_from_multi_buffers.call_count == 1
|
||||
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
|
||||
@@ -1180,8 +1203,7 @@ def test_store_sending_thread_group_id_excludes_physical_sharding():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
@@ -1210,8 +1232,7 @@ def test_store_sending_thread_multiple_segments_share_logical_group_id():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
@@ -1260,8 +1281,7 @@ def test_store_sending_thread_group_ids_share_across_kv_cache_groups():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_multi_group_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_multi_group_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == [
|
||||
@@ -1296,8 +1316,7 @@ def test_store_sending_thread_group_ids_follow_missing_key_filter():
|
||||
supports_group_ids=True,
|
||||
)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
|
||||
assert keys == ["test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131"]
|
||||
@@ -1866,15 +1885,15 @@ def test_store_sending_thread_clamps_token_len_to_lcm():
|
||||
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=33,
|
||||
block_ids=([0, 1, 2],),
|
||||
block_hashes=[b"a0", b"a1", b"a2"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
@@ -1905,20 +1924,19 @@ def test_store_sending_thread_skips_when_token_len_below_lcm():
|
||||
store, coord=coord, token_databases=[db], block_size=64
|
||||
)
|
||||
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=32,
|
||||
block_ids=([0, 1],),
|
||||
block_hashes=[b"a0", b"a1"],
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
store.batch_is_exist.assert_not_called()
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
assert thread.stored_requests["r0"] == 0
|
||||
|
||||
|
||||
def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
@@ -1982,15 +2000,15 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
@@ -2057,16 +2075,16 @@ def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
|
||||
)
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(8)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._saved_offset["r0"] = 32
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=64,
|
||||
block_ids=([0, 1], list(range(8))),
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
keys = store.batch_put_from_multi_buffers.call_args.args[0]
|
||||
@@ -2128,8 +2146,8 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
thread.enable_kv_event = True
|
||||
|
||||
hs = [bytes([i + 1]) * 4 for i in range(4)]
|
||||
thread.add_stored_request("r0")
|
||||
thread._handle_request(
|
||||
_run_store_req(
|
||||
thread,
|
||||
ReqMeta(
|
||||
req_id="r0",
|
||||
token_len_chunk=32,
|
||||
@@ -2137,7 +2155,7 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
|
||||
block_hashes=hs,
|
||||
can_save=True,
|
||||
token_ids=list(range(32)),
|
||||
)
|
||||
),
|
||||
)
|
||||
|
||||
full_event, swa_event = thread.get_kv_events()
|
||||
|
||||
Reference in New Issue
Block a user