[Bugfix][Mooncake] Release GPU pin on failed store in MooncakeStoreConnector (#43742)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-06-01 18:29:18 -07:00
committed by GitHub
co-authored by Claude
parent 517e74a964
commit d68f0b220e
2 changed files with 198 additions and 166 deletions
@@ -461,6 +461,35 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
assert store.batch_put_from_multi_buffers.call_count == 2 assert store.batch_put_from_multi_buffers.call_count == 2
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
# `batch_is_exist` raising must still decrement `stored_requests` so the
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
store = MagicMock()
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
with pytest.raises(RuntimeError):
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert thread.stored_requests["req-a"] == 0
store.batch_put_from_multi_buffers.assert_not_called()
def test_store_sending_thread_releases_pin_on_batch_put_failure():
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
# pin must still be released through the finally block.
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert thread.stored_requests["req-a"] == 0
def test_store_recving_thread_reports_failed_block_ids(): def test_store_recving_thread_reports_failed_block_ids():
store = MagicMock() store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, -5, -7] store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
@@ -517,18 +517,19 @@ class KVCacheStoreSendingThread(KVTransferThread):
if req_id not in self.stored_requests: if req_id not in self.stored_requests:
self.request_queue.task_done() self.request_queue.task_done()
return return
# Decrement the in-flight counter and signal task_done() in `finally`
# so the scheduler can release the GPU blocks it pinned for this
# request (via `delay_free_blocks`) even when the store path raises.
try:
if token_len == 0: if token_len == 0:
self.dec_stored_request(req_id)
self.request_queue.task_done()
return return
if self._should_skip_request(req_id): if self._should_skip_request(req_id):
logger.debug( logger.debug(
"Skipping Mooncake store for request %s while CPU/disk offloading " "Skipping Mooncake store for request %s while CPU/disk "
"is under pressure", "offloading is under pressure",
req_id, req_id,
) )
self.dec_stored_request(req_id)
self.request_queue.task_done()
return return
# Within each lcm region only per-spec relevant chunks are loaded # Within each lcm region only per-spec relevant chunks are loaded
@@ -561,7 +562,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
group_indices = group_indices[sl] group_indices = group_indices[sl]
if not keys: if not keys:
self.dec_stored_request(req_id)
return return
# Check which blocks already exist (dedup) # Check which blocks already exist (dedup)
@@ -582,10 +582,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
save_exists_start, save_exists_start,
len(keys), len(keys),
) )
missing_indices = [i for i, exists in enumerate(exists_states) if exists != 1] missing_indices = [
i for i, exists in enumerate(exists_states) if exists != 1
]
if not missing_indices: if not missing_indices:
self.dec_stored_request(req_id)
return return
starts = [starts[i] for i in missing_indices] starts = [starts[i] for i in missing_indices]
@@ -618,7 +619,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
if self.enable_kv_event: if self.enable_kv_event:
token_ids = ( token_ids = (
req_meta.token_ids[s:e] if req_meta.token_ids is not None else None req_meta.token_ids[s:e]
if req_meta.token_ids is not None
else None
) )
stored_event = BlockStored( stored_event = BlockStored(
block_hashes=[new_block_hashes[idx]], block_hashes=[new_block_hashes[idx]],
@@ -695,7 +698,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
if self.enable_kv_event and stored_events: if self.enable_kv_event and stored_events:
self.update_kv_event(stored_events) self.update_kv_event(stored_events)
finally:
self.dec_stored_request(req_id) self.dec_stored_request(req_id)
self.request_queue.task_done() self.request_queue.task_done()