[Bugfix][Mooncake] Reference GPU blocks for in-flight store jobs and key the store ledger by store_job_id (#52372)

Signed-off-by: Guanyi Chen <939416532@qq.com>
This commit is contained in:
Guanyi Chen
2026-08-16 16:09:46 -07:00
committed by GitHub
parent eee538d5da
commit dc9ae4b8ac
7 changed files with 477 additions and 253 deletions
@@ -228,12 +228,11 @@ def test_e2e_swa_plus_full_save_then_lookup_hits():
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=hs,
can_save=True,
store_job_id=1,
)
send_thread.add_stored_request("r0")
# Put the request in the queue so task_done() doesn't underflow.
send_thread.request_queue.put(save_req)
req = send_thread.request_queue.get()
send_thread._handle_request(req)
# add_request also queues the job, so task_done() doesn't underflow.
send_thread.add_request(save_req)
send_thread._handle_request(send_thread.request_queue.get())
# Point worker.store at the dict store (the worker constructor captured
# the MagicMock; replace with the real dict store for lookup).
@@ -499,15 +498,15 @@ def test_offload_syncs_event_before_put():
can_save=True,
num_prompt_tokens=12,
partial_tail_offloads=[(1, 7, 12)],
store_job_id=1,
)
req.current_event = event
send.add_stored_request("r1")
send.request_queue.put(req)
send.add_request(req)
send._handle_request(send.request_queue.get())
assert send.request_queue.qsize() == 0
assert store._data
assert send.stored_requests["r1"] == 0
assert send.stored_requests["r1"] == set()
event.synchronize.assert_called_once()
@@ -5,12 +5,14 @@ from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
MooncakeStoreWorkerMetadata,
ReqMeta,
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
MooncakeStoreScheduler,
)
from vllm.v1.core.block_pool import BlockPool
def _make_bare_scheduler(
@@ -27,6 +29,12 @@ def _make_bare_scheduler(
scheduler._unfinished_request_ids = {"req-0"}
scheduler._unfinished_requests = {}
scheduler._request_trackers = {}
scheduler._gpu_block_pool = BlockPool(
num_gpu_blocks=64, enable_caching=True, hash_block_size=hash_block_size
)
scheduler._num_workers = 1
scheduler._next_store_job_id = 0
scheduler._pinned_saves = {}
return scheduler
@@ -64,6 +72,14 @@ def _make_preemption_scheduler_output():
)
def _make_worker_output(completed_saves: dict[int, int]) -> SimpleNamespace:
return SimpleNamespace(
kv_connector_worker_meta=MooncakeStoreWorkerMetadata(
completed_saves=completed_saves
)
)
def _add_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
@@ -133,7 +149,7 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
assert tracker.num_saved_tokens == 48
def test_preemption_resets_tracker_before_request_finished():
def test_preemption_resets_tracker():
scheduler = _make_bare_scheduler()
_add_unfinished_request(
scheduler,
@@ -152,8 +168,6 @@ def test_preemption_resets_tracker_before_request_finished():
assert tracker.token_ids is None
assert tracker.has_pending_offload is False
assert tracker.prefill_end_tokens == 0
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0, 1],)) == (False, None)
def test_preemption_clears_stale_load_state():
@@ -215,7 +229,7 @@ def test_pending_load_does_not_co_queue_save():
# enqueue a save in the same scheduling step. Co-queuing both produces a
# recv+send pair for the same req_id, and the scheduler's
# _update_from_kv_xfer_finished then trips `assert req_id in self.requests`
# when both completions land for the delay-freed request.
# when a completion lands for a request it has already dropped.
scheduler = _make_bare_scheduler()
_make_pending_load_unfinished_request(
scheduler,
@@ -238,9 +252,7 @@ def test_pending_load_does_not_co_queue_save():
# Load is still issued as planned.
assert req_meta.load_spec is not None
assert req_meta.load_spec.can_load is True
# And the tracker's saved-tokens watermark stays at 0 so request_finished
# later sees `num_saved_tokens <= 0` and frees immediately rather than
# waiting for a finished_sending that will never come.
# And the save watermark does not advance for a save that was never queued.
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
@@ -661,30 +673,34 @@ def test_disabled_lookup_reports_no_hit_without_querying_client():
assert scheduler.load_specs == {}
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
# the worker skips the normal save). Pending-offload state delays the free
# without advancing the normal-save watermark before the put succeeds.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
def _add_pending_partial_tail_request(
scheduler: MooncakeStoreScheduler,
*,
num_tokens: int,
block_hashes: list[bytes],
block_ids: tuple[list[int], ...],
) -> SimpleNamespace:
"""Register a sub-block request and return the step that offloads its tail.
The CoW block holding the tail is block 7, which the core deliberately keeps
out of the request's block table.
"""
request = SimpleNamespace(
all_token_ids=list(range(12)),
block_hashes=[b"h0", b"h1", b"h2"],
all_token_ids=list(range(num_tokens)),
block_hashes=block_hashes,
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0],))
scheduler._unfinished_requests["req-0"] = (request, block_ids)
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=12,
allocated_block_ids=([0],),
token_len=num_tokens,
allocated_block_ids=block_ids,
num_saved_tokens=0,
token_ids=list(range(12)),
prefill_end_tokens=12,
token_ids=list(range(num_tokens)),
prefill_end_tokens=num_tokens,
)
out = SimpleNamespace(
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
@@ -699,6 +715,21 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
partial_tail_offloads={"req-0": [(1, 7, 12)]},
)
def test_pending_partial_tail_emits_offload_only_reqmeta():
# A sub-block prompt never produces a block-aligned save, so the partial-
# tail offload arriving this step is emitted as an offload-only ReqMeta
# (can_save=True so it takes the normal enqueue path, token_len_chunk=0 so
# the worker skips the normal save), without advancing the normal-save
# watermark before the put succeeds.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([0],),
)
meta = scheduler.build_connector_meta(out)
assert len(meta.requests) == 1
@@ -712,42 +743,16 @@ def test_pending_partial_tail_emits_offload_only_reqmeta():
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
assert tracker.has_pending_offload is True
request = SimpleNamespace(request_id="req-0")
assert scheduler.request_finished(request, ([0],)) == (True, None)
def test_resumed_partial_tail_uses_handoff_boundary():
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
request = SimpleNamespace(
all_token_ids=list(range(20)),
# Resumption replays prompt + previously generated tokens.
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=20,
block_hashes=[b"h0", b"h1", b"h2", b"h3", b"h4"],
num_output_placeholders=0,
num_prompt_tokens=12,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
scheduler._request_trackers["req-0"] = RequestTracker(
req_id="req-0",
token_len=20,
allocated_block_ids=([0, 1],),
num_saved_tokens=0,
token_ids=list(range(20)),
# Resumption replays prompt + previously generated tokens.
prefill_end_tokens=20,
)
out = SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
resumed_req_ids=set(),
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
partial_tail_offloads={"req-0": [(1, 7, 12)]},
block_ids=([0, 1],),
)
meta = scheduler.build_connector_meta(out)
@@ -791,3 +796,69 @@ def test_resumed_partial_tail_attached_to_save_keeps_handoff_boundary():
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 48
assert tracker.has_pending_offload is True
def test_partial_tail_cow_block_is_referenced_for_the_job():
# The CoW block a partial-tail offload reads is deliberately kept out of the
# request block table, so it is absent from ReqMeta.block_ids. The worker
# DMAs out of it just as asynchronously, so it needs its own reference.
scheduler = _make_bare_scheduler(hash_block_size=4, enable_partial_hash_hits=True)
out = _add_pending_partial_tail_request(
scheduler,
num_tokens=12,
block_hashes=[b"h0", b"h1", b"h2"],
block_ids=([0],),
)
pool = scheduler._gpu_block_pool
meta = scheduler.build_connector_meta(out)
store_job_id = meta.requests[0].store_job_id
# It leads the list, as in `pop_blocks_for_free`, so that the reversed free
# puts it last in eviction priority.
assert scheduler._pinned_saves[store_job_id][0] == [7, 0]
assert pool.blocks[7].ref_cnt == 1
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert pool.blocks[7].ref_cnt == 0
def test_store_job_blocks_are_released_once_every_rank_reports():
# Every rank DMAs the job's blocks on its own, so the reference can only be
# dropped once the last of them reports. Until then the engine has to keep
# stepping: a completion only reaches the scheduler as worker metadata
# attached to a step, and a finishing request no longer defers its own free.
scheduler = _make_bare_scheduler()
scheduler._num_workers = 2
_add_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
prefill_end_tokens=48,
)
pool = scheduler._gpu_block_pool
assert scheduler.has_pending_push_work() is False
meta = scheduler.build_connector_meta(
_make_scheduler_output(scheduled_spec_tokens=None)
)
store_job_id = meta.requests[0].store_job_id
assert pool.blocks[2].ref_cnt == 1
assert scheduler.has_pending_push_work() is True
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert pool.blocks[2].ref_cnt == 1
assert scheduler.has_pending_push_work() is True
scheduler.update_connector_output(_make_worker_output({store_job_id: 1}))
assert pool.blocks[2].ref_cnt == 0
assert scheduler.has_pending_push_work() is False
def test_worker_metadata_aggregates_completions_across_ranks():
# The engine merges each rank's metadata before the scheduler sees it, so a
# job that every rank finished in one step arrives as a single count.
merged = MooncakeStoreWorkerMetadata(completed_saves={1: 1}).aggregate(
MooncakeStoreWorkerMetadata(completed_saves={1: 1, 2: 1})
)
assert merged.completed_saves == {1: 2, 2: 1}
@@ -1,6 +1,8 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import contextlib
import itertools
import json
import logging
import math
@@ -156,6 +158,17 @@ def _make_load_req(
)
_TEST_SAVE_SEQ = itertools.count(1)
def _run_store_req(thread, req_meta: ReqMeta) -> None:
"""Register, enqueue and run a store job the way the worker does."""
if req_meta.store_job_id is None:
req_meta.store_job_id = next(_TEST_SAVE_SEQ)
thread.add_request(req_meta)
thread._handle_request(thread.request_queue.get())
def _make_store_req(req_id: str, block_hashes: list[bytes]) -> ReqMeta:
return ReqMeta(
req_id=req_id,
@@ -385,27 +398,23 @@ def test_store_sending_thread_skips_request_during_cpu_pressure():
]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert thread._store_pressure_active is True
assert "req-a" in thread._skip_store_requests
assert store.batch_put_from_multi_buffers.call_count == 1
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
assert store.batch_put_from_multi_buffers.call_count == 1
thread.add_stored_request("req-b")
thread._handle_request(_make_store_req("req-b", [b"b0", b"b1"]))
_run_store_req(thread, _make_store_req("req-b", [b"b0", b"b1"]))
assert thread._store_pressure_active is False
assert "req-a" not in thread._skip_store_requests
assert store.batch_put_from_multi_buffers.call_count == 2
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a4", b"a5"]))
_run_store_req(thread, _make_store_req("req-a", [b"a4", b"a5"]))
assert store.batch_put_from_multi_buffers.call_count == 3
@@ -418,8 +427,7 @@ def test_store_sending_thread_records_mooncake_metrics():
stats = MooncakeStoreConnectorStats()
thread._record_operation_cb = stats.record_operation
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert len(stats.data["save_exists"]) == 1
assert stats.data["save_exists"][0]["num_keys"] == 2
@@ -497,16 +505,16 @@ def test_store_sending_thread_delta_saves_only_new_full_attention_chunks():
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -523,16 +531,16 @@ def test_store_sending_thread_delta_strides_with_local_phase():
store.batch_put_from_multi_buffers.return_value = [256]
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -550,15 +558,15 @@ def test_tp_sharded_group_saves_every_block_on_every_rank():
thread = _make_store_sending_thread(store, tp_rank=0, put_step=2)
thread.group_put_steps = [1]
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -576,15 +584,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=16,
block_ids=([0],),
block_hashes=[b"a0"],
can_save=True,
)
),
)
store.batch_is_exist.assert_not_called()
@@ -595,15 +603,15 @@ def test_store_sending_thread_retries_skipped_range_after_pressure():
# The next batch resumes from offset 0, re-covering the chunk skipped under
# pressure (chunk 0) rather than losing it.
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -703,13 +711,11 @@ def test_partial_tail_offload_honors_active_pressure_gate():
thread = _make_partial_tail_send_thread(store)
thread._store_pressure_active = True
thread._skip_store_requests.add("req-a")
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["req-a"] == 0
def test_partial_tail_put_failure_activates_pressure_gate():
@@ -717,17 +723,14 @@ def test_partial_tail_put_failure_activates_pressure_gate():
store.batch_is_exist.side_effect = lambda keys: [0] * len(keys)
store.batch_put_from_multi_buffers.return_value = [256, -200, 256]
thread = _make_partial_tail_send_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
assert thread._store_pressure_active is True
assert thread._skip_store_requests == {"req-a"}
assert thread._saved_offset.get("req-a", 0) == 0
assert thread.stored_requests["req-a"] == 0
thread.add_stored_request("req-a")
thread._handle_request(_make_partial_tail_req([1, 2, 3]))
_run_store_req(thread, _make_partial_tail_req([1, 2, 3]))
assert store.batch_put_from_multi_buffers.call_count == 1
@@ -737,16 +740,16 @@ def test_store_sending_thread_delta_start_rank_saves_second_local_chunk():
store.batch_put_from_multi_buffers.return_value = [256, 256]
thread = _make_store_sending_thread(store, tp_rank=1, put_step=2)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 16
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3],),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -790,16 +793,16 @@ def test_store_sending_thread_delta_saves_only_new_masked_chunks():
token_databases=[db_full, db_masked],
)
thread.add_stored_request("req-a")
thread._saved_offset["req-a"] = 32
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=64,
block_ids=([0, 1, 2, 3], [0, 1, 2, 3]),
block_hashes=[b"a0", b"a1", b"a2", b"a3"],
can_save=True,
)
),
)
keys = store.batch_is_exist.call_args.args[0]
@@ -845,15 +848,15 @@ def test_store_sending_thread_prepares_missing_chunks_once_per_group():
coord=coord,
token_databases=[db0, db1],
)
thread.add_stored_request("req-a")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="req-a",
token_len_chunk=48,
block_ids=([0, 1, 2], [2, 1, 0]),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
)
),
)
db0.prepare_value.assert_not_called()
@@ -881,46 +884,71 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
]
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert thread._store_pressure_active is False
assert "req-a" not in thread._skip_store_requests
assert store.batch_put_from_multi_buffers.call_count == 1
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a2", b"a3"]))
_run_store_req(thread, _make_store_req("req-a", [b"a2", b"a3"]))
assert store.batch_put_from_multi_buffers.call_count == 2
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
# `batch_is_exist` raising must still decrement `stored_requests` so the
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
store = MagicMock()
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
with pytest.raises(RuntimeError):
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert thread.stored_requests["req-a"] == 0
store.batch_put_from_multi_buffers.assert_not_called()
def test_store_sending_thread_releases_pin_on_batch_put_failure():
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
# pin must still be released through the finally block.
@pytest.mark.parametrize(
"failing_call", ["batch_is_exist", "batch_put_from_multi_buffers"]
)
def test_store_sending_thread_reports_job_when_store_raises(failing_call):
# A store that blows up must still report its job, or the scheduler keeps
# the job's GPU block references for the rest of the run.
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
getattr(store, failing_call).side_effect = RuntimeError("mooncake down")
thread = _make_store_sending_thread(store)
req = _make_store_req("req-a", [b"a0", b"a1"])
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
with contextlib.suppress(RuntimeError):
_run_store_req(thread, req)
assert thread.stored_requests["req-a"] == 0
assert thread.take_completed_saves() == {req.store_job_id: 1}
def test_store_sending_thread_reports_job_when_the_preamble_raises():
# The report has to survive a failure before the first store call too, so
# every dequeue leaves through the same exit.
thread = _make_store_sending_thread(MagicMock())
req = _make_store_req("req-a", [b"a0", b"a1"])
req.token_len_chunk = None # type: ignore[assignment]
with contextlib.suppress(TypeError):
_run_store_req(thread, req)
assert thread.take_completed_saves() == {req.store_job_id: 1}
thread.request_queue.task_done.assert_called_once()
def test_stale_store_job_cannot_touch_a_reused_request_id():
# A preempted request resumes under its original id, so a job left over from
# the retired generation carries a req_id that now belongs to a live one.
thread = _make_store_sending_thread(MagicMock())
stale = _make_store_req("req-a", [b"a0", b"a1"])
stale.store_job_id = 1
thread.add_request(stale)
thread._record_saved(stale, 32)
thread.delete_finished_stored_request("req-a")
live = _make_store_req("req-a", [b"a0", b"a1"])
live.store_job_id = 2
thread.add_request(live)
thread.finish_store_job(stale)
thread._record_saved(stale, 64)
thread._mark_request_skipped_for_pressure(stale)
assert thread.is_live_store_job(live)
assert not thread.is_live_store_job(stale)
assert thread._saved_offset.get("req-a") is None
assert "req-a" not in thread._skip_store_requests
def test_store_recving_thread_reports_failed_block_ids():
@@ -991,8 +1019,7 @@ def test_store_sending_thread_passes_replicate_config_when_preferred_segment_set
replicate_config = SimpleNamespace(preferred_segment="10.0.0.7:50053")
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
call_args = store.batch_put_from_multi_buffers.call_args.args
@@ -1010,8 +1037,7 @@ def test_store_sending_thread_passes_default_replicate_config_when_no_preferred_
replicate_config = SimpleNamespace()
thread = _make_store_sending_thread(store, replicate_config=replicate_config)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
call_args = store.batch_put_from_multi_buffers.call_args.args
@@ -1067,8 +1093,7 @@ def test_store_sending_thread_sets_group_ids_when_enabled():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
@@ -1092,8 +1117,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_flag_disabled():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
@@ -1112,8 +1136,7 @@ def test_store_sending_thread_leaves_group_ids_unchanged_when_unsupported():
supports_group_ids=False,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
assert store.batch_put_from_multi_buffers.call_count == 1
assert store.batch_put_from_multi_buffers.call_args.args[3] is replicate_config
@@ -1180,8 +1203,7 @@ def test_store_sending_thread_group_id_excludes_physical_sharding():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1210,8 +1232,7 @@ def test_store_sending_thread_multiple_segments_share_logical_group_id():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1260,8 +1281,7 @@ def test_store_sending_thread_group_ids_share_across_kv_cache_groups():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_multi_group_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_multi_group_store_req("req-a", [b"a0", b"a1"]))
keys, addrs, sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == [
@@ -1296,8 +1316,7 @@ def test_store_sending_thread_group_ids_follow_missing_key_filter():
supports_group_ids=True,
)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
_run_store_req(thread, _make_store_req("req-a", [b"a0", b"a1"]))
keys, _addrs, _sizes, config = store.batch_put_from_multi_buffers.call_args.args
assert keys == ["test-model@tp_rank:0@pcp0@dcp0@pp_rank:0@group:0@6131"]
@@ -1866,15 +1885,15 @@ def test_store_sending_thread_clamps_token_len_to_lcm():
# token_len_chunk=33 clamps to 32 → 2 chunks (not 3 with a partial 1-token chunk).
thread = _make_store_sending_thread(store)
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=33,
block_ids=([0, 1, 2],),
block_hashes=[b"a0", b"a1", b"a2"],
can_save=True,
)
),
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
@@ -1905,20 +1924,19 @@ def test_store_sending_thread_skips_when_token_len_below_lcm():
store, coord=coord, token_databases=[db], block_size=64
)
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=32,
block_ids=([0, 1],),
block_hashes=[b"a0", b"a1"],
can_save=True,
)
),
)
store.batch_is_exist.assert_not_called()
store.batch_put_from_multi_buffers.assert_not_called()
assert thread.stored_requests["r0"] == 0
def test_store_sending_thread_only_stores_swa_blocks_in_window():
@@ -1982,15 +2000,15 @@ def test_store_sending_thread_only_stores_swa_blocks_in_window():
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
)
),
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
@@ -2057,16 +2075,16 @@ def test_store_sending_thread_delta_saves_only_new_swa_boundary_chunks():
)
hs = [bytes([i + 1]) * 4 for i in range(8)]
thread.add_stored_request("r0")
thread._saved_offset["r0"] = 32
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=64,
block_ids=([0, 1], list(range(8))),
block_hashes=hs,
can_save=True,
)
),
)
keys = store.batch_put_from_multi_buffers.call_args.args[0]
@@ -2128,8 +2146,8 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
thread.enable_kv_event = True
hs = [bytes([i + 1]) * 4 for i in range(4)]
thread.add_stored_request("r0")
thread._handle_request(
_run_store_req(
thread,
ReqMeta(
req_id="r0",
token_len_chunk=32,
@@ -2137,7 +2155,7 @@ def test_store_sending_thread_kv_events_use_group_chunk_metadata():
block_hashes=hs,
can_save=True,
token_ids=list(range(32)),
)
),
)
full_event, swa_event = thread.get_kv_events()