[Bugfix][Mooncake] Release GPU pin on failed store in MooncakeStoreConnector (#43742)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -461,6 +461,35 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||
|
||||
|
||||
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
|
||||
# `batch_is_exist` raising must still decrement `stored_requests` so the
|
||||
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
with pytest.raises(RuntimeError):
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
store.batch_put_from_multi_buffers.assert_not_called()
|
||||
|
||||
|
||||
def test_store_sending_thread_releases_pin_on_batch_put_failure():
|
||||
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
|
||||
# pin must still be released through the finally block.
|
||||
store = MagicMock()
|
||||
store.batch_is_exist.return_value = [0, 0]
|
||||
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
|
||||
thread = _make_store_sending_thread(store)
|
||||
|
||||
thread.add_stored_request("req-a")
|
||||
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||
|
||||
assert thread.stored_requests["req-a"] == 0
|
||||
|
||||
|
||||
def test_store_recving_thread_reports_failed_block_ids():
|
||||
store = MagicMock()
|
||||
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
|
||||
|
||||
Reference in New Issue
Block a user