[Bugfix][Mooncake] Release GPU pin on failed store in MooncakeStoreConnector (#43742)

Signed-off-by: Dao Le <Dao007forever@gmail.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Dao007forever
2026-06-01 18:29:18 -07:00
committed by GitHub
co-authored by Claude
parent 517e74a964
commit d68f0b220e
2 changed files with 198 additions and 166 deletions
@@ -461,6 +461,35 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
assert store.batch_put_from_multi_buffers.call_count == 2
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
# `batch_is_exist` raising must still decrement `stored_requests` so the
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
store = MagicMock()
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
with pytest.raises(RuntimeError):
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert thread.stored_requests["req-a"] == 0
store.batch_put_from_multi_buffers.assert_not_called()
def test_store_sending_thread_releases_pin_on_batch_put_failure():
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
# pin must still be released through the finally block.
store = MagicMock()
store.batch_is_exist.return_value = [0, 0]
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
thread = _make_store_sending_thread(store)
thread.add_stored_request("req-a")
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
assert thread.stored_requests["req-a"] == 0
def test_store_recving_thread_reports_failed_block_ids():
store = MagicMock()
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]