[KV Connector] MooncakeStore: don't co-queue save with load to avoid double delayed-free (#43371)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-05-22 16:14:11 +00:00
committed by GitHub
co-authored by Claude Opus 4.7
parent c7624bea5e
commit b21f3d56d4
2 changed files with 241 additions and 0 deletions
@@ -4,6 +4,8 @@
from types import SimpleNamespace
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.data import (
LoadSpec,
ReqMeta,
RequestTracker,
)
from vllm.distributed.kv_transfer.kv_connector.v1.mooncake.store.scheduler import (
@@ -109,3 +111,236 @@ def test_cached_request_without_spec_decode_keeps_current_step_save_overlap():
tracker = scheduler._request_trackers["req-0"]
assert tracker.token_len == 48
assert tracker.num_saved_tokens == 48
def _make_pending_load_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
num_tokens: int,
block_hashes: list[bytes],
block_ids: tuple[list[int], ...] = ([0, 1, 2],),
) -> None:
request = SimpleNamespace(
num_tokens=num_tokens,
block_hashes=block_hashes,
num_output_placeholders=0,
)
scheduler._unfinished_requests["req-0"] = (request, block_ids)
def _make_pending_load_scheduler_output() -> SimpleNamespace:
"""scheduler_output for a step where req-0 is parked on a pending load
(not in scheduled_new_reqs or scheduled_cached_reqs)."""
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=[],
new_block_ids=[],
num_computed_tokens=[],
),
num_scheduled_tokens={},
scheduled_spec_decode_tokens={},
)
def test_pending_load_does_not_co_queue_save():
# Regression: a cache-hit request waiting on an async load must not also
# enqueue a save in the same scheduling step. Co-queuing both produces a
# recv+send pair for the same req_id, and the scheduler's
# _update_from_kv_xfer_finished then trips `assert req_id in self.requests`
# when both completions land for the delay-freed request.
scheduler = _make_bare_scheduler()
_make_pending_load_unfinished_request(
scheduler,
num_tokens=48,
block_hashes=[b"h0", b"h1", b"h2"],
)
scheduler.load_specs["req-0"] = LoadSpec(
vllm_cached_tokens=0,
kvpool_cached_tokens=48,
can_load=True,
)
meta = scheduler.build_connector_meta(_make_pending_load_scheduler_output())
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
# Save must be off so the worker does not call add_stored_request.
assert req_meta.can_save is False
# Load is still issued as planned.
assert req_meta.load_spec is not None
assert req_meta.load_spec.can_load is True
# And the tracker's saved-tokens watermark stays at 0 so request_finished
# later sees `num_saved_tokens <= 0` and frees immediately rather than
# waiting for a finished_sending that will never come.
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
def _make_resumed_unfinished_request(
scheduler: MooncakeStoreScheduler,
*,
token_ids: list[int],
block_hashes: list[bytes],
num_computed_tokens: int,
) -> None:
request = SimpleNamespace(
all_token_ids=token_ids,
block_hashes=block_hashes,
num_computed_tokens=num_computed_tokens,
num_output_placeholders=0,
)
scheduler._unfinished_requests["req-0"] = (request, ([0, 1],))
def _make_resumed_scheduler_output(*, num_scheduled_tokens: int) -> SimpleNamespace:
return SimpleNamespace(
finished_req_ids=set(),
preempted_req_ids=set(),
scheduled_new_reqs=[],
scheduled_cached_reqs=SimpleNamespace(
req_ids=["req-0"],
new_block_ids=[([2],)],
num_computed_tokens=[0],
),
num_scheduled_tokens={"req-0": num_scheduled_tokens},
scheduled_spec_decode_tokens={},
)
def test_resumed_from_preemption_with_load_skips_save():
# On resume-from-preemption with a cache hit, the same co-queueing race
# applies: the resumed-from-preemption branch in build_connector_meta also
# passes load_spec.can_load=True. Skip save in this step; subsequent
# cached_reqs steps will save new tokens normally.
scheduler = _make_bare_scheduler()
scheduler._preempted_req_ids = {"req-0"}
_make_resumed_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
num_computed_tokens=0,
)
scheduler.load_specs["req-0"] = LoadSpec(
vllm_cached_tokens=0,
kvpool_cached_tokens=48,
can_load=True,
)
meta = scheduler.build_connector_meta(
_make_resumed_scheduler_output(num_scheduled_tokens=48)
)
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
assert req_meta.can_save is False
assert req_meta.load_spec is not None
assert req_meta.load_spec.can_load is True
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 0
def test_resumed_from_preemption_without_load_still_saves():
# No load_spec → behavior is unchanged: save proceeds.
scheduler = _make_bare_scheduler()
scheduler._preempted_req_ids = {"req-0"}
_make_resumed_unfinished_request(
scheduler,
token_ids=list(range(48)),
block_hashes=[b"h0", b"h1", b"h2"],
num_computed_tokens=0,
)
meta = scheduler.build_connector_meta(
_make_resumed_scheduler_output(num_scheduled_tokens=48)
)
assert len(meta.requests) == 1
req_meta = meta.requests[0]
assert req_meta.req_id == "req-0"
assert req_meta.can_save is True
assert req_meta.load_spec is None
tracker = scheduler._request_trackers["req-0"]
assert tracker.num_saved_tokens == 48
# Focused tests for ReqMeta.from_request_tracker — the centralized guard that
# enforces "a ReqMeta never carries both a save and a load".
def test_from_request_tracker_load_overrides_caller_skip_save():
# Caller asks for skip_save=False, but load_spec.can_load=True. The
# function must force skip_save=True to avoid producing a ReqMeta the
# worker would enqueue on both kv_send_thread and kv_recv_thread.
tracker = RequestTracker(
req_id="req-0",
token_len=48,
allocated_block_ids=([0, 1, 2],),
num_saved_tokens=0,
)
load_spec = LoadSpec(vllm_cached_tokens=0, kvpool_cached_tokens=48, can_load=True)
req_meta = ReqMeta.from_request_tracker(
tracker,
block_size=16,
load_spec=load_spec,
skip_save=False,
block_hashes=[b"h0", b"h1", b"h2"],
)
assert req_meta is not None
assert req_meta.can_save is False
assert req_meta.load_spec is load_spec
assert tracker.num_saved_tokens == 0
def test_from_request_tracker_load_with_can_load_false_still_saves():
# A LoadSpec with can_load=False (e.g., no external tokens to load after
# update_state_after_alloc) must not suppress the save.
tracker = RequestTracker(
req_id="req-0",
token_len=48,
allocated_block_ids=([0, 1, 2],),
num_saved_tokens=0,
)
load_spec = LoadSpec(vllm_cached_tokens=0, kvpool_cached_tokens=48, can_load=False)
req_meta = ReqMeta.from_request_tracker(
tracker,
block_size=16,
load_spec=load_spec,
skip_save=False,
block_hashes=[b"h0", b"h1", b"h2"],
)
assert req_meta is not None
assert req_meta.can_save is True
# from_request_tracker clears load_spec when can_load is False.
assert req_meta.load_spec is None
assert tracker.num_saved_tokens == 48
def test_from_request_tracker_no_load_saves_normally():
tracker = RequestTracker(
req_id="req-0",
token_len=48,
allocated_block_ids=([0, 1, 2],),
num_saved_tokens=0,
)
req_meta = ReqMeta.from_request_tracker(
tracker,
block_size=16,
load_spec=None,
skip_save=False,
block_hashes=[b"h0", b"h1", b"h2"],
)
assert req_meta is not None
assert req_meta.can_save is True
assert req_meta.load_spec is None
assert tracker.num_saved_tokens == 48
@@ -236,6 +236,12 @@ class ReqMeta:
)
skip_save = skip_save or num_tokens_to_save < chunk_boundary
# A ReqMeta must never carry both a save AND a load.
# The save would also be wasted work — the bytes are being looked up
# in the store right now. Later cached_reqs steps save new tokens
# normally.
if load_spec is not None and load_spec.can_load:
skip_save = True
if skip_save and load_spec is None:
return None