[Bugfix][Mooncake] Release GPU pin on failed store in MooncakeStoreConnector (#43742)
Signed-off-by: Dao Le <Dao007forever@gmail.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -461,6 +461,35 @@ def test_store_sending_thread_only_skips_on_no_available_handle():
|
|||||||
assert store.batch_put_from_multi_buffers.call_count == 2
|
assert store.batch_put_from_multi_buffers.call_count == 2
|
||||||
|
|
||||||
|
|
||||||
|
def test_store_sending_thread_releases_pin_on_batch_is_exist_failure():
|
||||||
|
# `batch_is_exist` raising must still decrement `stored_requests` so the
|
||||||
|
# scheduler can drop `delay_free_blocks` and release the pinned GPU blocks.
|
||||||
|
store = MagicMock()
|
||||||
|
store.batch_is_exist.side_effect = RuntimeError("mooncake down")
|
||||||
|
thread = _make_store_sending_thread(store)
|
||||||
|
|
||||||
|
thread.add_stored_request("req-a")
|
||||||
|
with pytest.raises(RuntimeError):
|
||||||
|
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||||
|
|
||||||
|
assert thread.stored_requests["req-a"] == 0
|
||||||
|
store.batch_put_from_multi_buffers.assert_not_called()
|
||||||
|
|
||||||
|
|
||||||
|
def test_store_sending_thread_releases_pin_on_batch_put_failure():
|
||||||
|
# `batch_put_from_multi_buffers` raising is logged (not re-raised), and the
|
||||||
|
# pin must still be released through the finally block.
|
||||||
|
store = MagicMock()
|
||||||
|
store.batch_is_exist.return_value = [0, 0]
|
||||||
|
store.batch_put_from_multi_buffers.side_effect = RuntimeError("rdma error")
|
||||||
|
thread = _make_store_sending_thread(store)
|
||||||
|
|
||||||
|
thread.add_stored_request("req-a")
|
||||||
|
thread._handle_request(_make_store_req("req-a", [b"a0", b"a1"]))
|
||||||
|
|
||||||
|
assert thread.stored_requests["req-a"] == 0
|
||||||
|
|
||||||
|
|
||||||
def test_store_recving_thread_reports_failed_block_ids():
|
def test_store_recving_thread_reports_failed_block_ids():
|
||||||
store = MagicMock()
|
store = MagicMock()
|
||||||
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
|
store.batch_get_into_multi_buffers.return_value = [256, -5, -7]
|
||||||
|
|||||||
@@ -517,18 +517,19 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
|||||||
if req_id not in self.stored_requests:
|
if req_id not in self.stored_requests:
|
||||||
self.request_queue.task_done()
|
self.request_queue.task_done()
|
||||||
return
|
return
|
||||||
|
|
||||||
|
# Decrement the in-flight counter and signal task_done() in `finally`
|
||||||
|
# so the scheduler can release the GPU blocks it pinned for this
|
||||||
|
# request (via `delay_free_blocks`) even when the store path raises.
|
||||||
|
try:
|
||||||
if token_len == 0:
|
if token_len == 0:
|
||||||
self.dec_stored_request(req_id)
|
|
||||||
self.request_queue.task_done()
|
|
||||||
return
|
return
|
||||||
if self._should_skip_request(req_id):
|
if self._should_skip_request(req_id):
|
||||||
logger.debug(
|
logger.debug(
|
||||||
"Skipping Mooncake store for request %s while CPU/disk offloading "
|
"Skipping Mooncake store for request %s while CPU/disk "
|
||||||
"is under pressure",
|
"offloading is under pressure",
|
||||||
req_id,
|
req_id,
|
||||||
)
|
)
|
||||||
self.dec_stored_request(req_id)
|
|
||||||
self.request_queue.task_done()
|
|
||||||
return
|
return
|
||||||
|
|
||||||
# Within each lcm region only per-spec relevant chunks are loaded
|
# Within each lcm region only per-spec relevant chunks are loaded
|
||||||
@@ -561,7 +562,6 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
|||||||
group_indices = group_indices[sl]
|
group_indices = group_indices[sl]
|
||||||
|
|
||||||
if not keys:
|
if not keys:
|
||||||
self.dec_stored_request(req_id)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
# Check which blocks already exist (dedup)
|
# Check which blocks already exist (dedup)
|
||||||
@@ -582,10 +582,11 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
|||||||
save_exists_start,
|
save_exists_start,
|
||||||
len(keys),
|
len(keys),
|
||||||
)
|
)
|
||||||
missing_indices = [i for i, exists in enumerate(exists_states) if exists != 1]
|
missing_indices = [
|
||||||
|
i for i, exists in enumerate(exists_states) if exists != 1
|
||||||
|
]
|
||||||
|
|
||||||
if not missing_indices:
|
if not missing_indices:
|
||||||
self.dec_stored_request(req_id)
|
|
||||||
return
|
return
|
||||||
|
|
||||||
starts = [starts[i] for i in missing_indices]
|
starts = [starts[i] for i in missing_indices]
|
||||||
@@ -618,7 +619,9 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
|||||||
|
|
||||||
if self.enable_kv_event:
|
if self.enable_kv_event:
|
||||||
token_ids = (
|
token_ids = (
|
||||||
req_meta.token_ids[s:e] if req_meta.token_ids is not None else None
|
req_meta.token_ids[s:e]
|
||||||
|
if req_meta.token_ids is not None
|
||||||
|
else None
|
||||||
)
|
)
|
||||||
stored_event = BlockStored(
|
stored_event = BlockStored(
|
||||||
block_hashes=[new_block_hashes[idx]],
|
block_hashes=[new_block_hashes[idx]],
|
||||||
@@ -695,7 +698,7 @@ class KVCacheStoreSendingThread(KVTransferThread):
|
|||||||
|
|
||||||
if self.enable_kv_event and stored_events:
|
if self.enable_kv_event and stored_events:
|
||||||
self.update_kv_event(stored_events)
|
self.update_kv_event(stored_events)
|
||||||
|
finally:
|
||||||
self.dec_stored_request(req_id)
|
self.dec_stored_request(req_id)
|
||||||
self.request_queue.task_done()
|
self.request_queue.task_done()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user