[Bugfix][KV Offload] Centralize shared mmap cleanup in CPU worker (#51622)
Signed-off-by: Alex <jihui.huang@daocloud.io>
This commit is contained in:
@@ -1,8 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -46,6 +48,177 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None:
|
||||
"""Both directions drain before the worker releases its shared region."""
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
calls: list[str] = []
|
||||
|
||||
def record_store_shutdown() -> bool:
|
||||
calls.append("store")
|
||||
return True
|
||||
|
||||
def record_load_shutdown() -> bool:
|
||||
calls.append("load")
|
||||
return True
|
||||
|
||||
store_handler = MagicMock()
|
||||
load_handler = MagicMock()
|
||||
store_handler.shutdown.side_effect = record_store_shutdown
|
||||
load_handler.shutdown.side_effect = record_load_shutdown
|
||||
|
||||
def record_region_cleanup(**_: bool) -> bool:
|
||||
calls.append("region")
|
||||
return True
|
||||
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.cleanup.side_effect = record_region_cleanup
|
||||
worker._store_handler = store_handler
|
||||
worker._load_handler = load_handler
|
||||
worker._mmap_region = mmap_region
|
||||
|
||||
worker.shutdown()
|
||||
|
||||
assert calls == ["store", "load", "region"]
|
||||
store_handler.shutdown.assert_called_once_with()
|
||||
load_handler.shutdown.assert_called_once_with()
|
||||
mmap_region.cleanup.assert_called_once_with()
|
||||
assert worker._mmap_region is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failing_handler", ["store", "load"])
|
||||
def test_worker_logs_handler_error_and_cleans_region(
|
||||
caplog_vllm, monkeypatch: pytest.MonkeyPatch, failing_handler
|
||||
) -> None:
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
calls: list[str] = []
|
||||
store_handler = MagicMock()
|
||||
load_handler = MagicMock()
|
||||
if failing_handler == "store":
|
||||
|
||||
def fail_store_shutdown() -> bool:
|
||||
calls.append("store")
|
||||
raise RuntimeError("transfer did not drain")
|
||||
|
||||
store_handler.shutdown.side_effect = fail_store_shutdown
|
||||
else:
|
||||
|
||||
def fail_load_shutdown() -> bool:
|
||||
calls.append("load")
|
||||
raise RuntimeError("transfer did not drain")
|
||||
|
||||
load_handler.shutdown.side_effect = fail_load_shutdown
|
||||
|
||||
def record_other_shutdown() -> bool:
|
||||
calls.append("load" if failing_handler == "store" else "store")
|
||||
return True
|
||||
|
||||
if failing_handler == "store":
|
||||
load_handler.shutdown.side_effect = record_other_shutdown
|
||||
else:
|
||||
store_handler.shutdown.side_effect = record_other_shutdown
|
||||
|
||||
def record_device_sync() -> None:
|
||||
calls.append("sync")
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
|
||||
|
||||
def record_region_cleanup() -> None:
|
||||
calls.append("region")
|
||||
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.cleanup.side_effect = record_region_cleanup
|
||||
worker._store_handler = store_handler
|
||||
worker._load_handler = load_handler
|
||||
worker._mmap_region = mmap_region
|
||||
|
||||
with caplog_vllm.at_level(
|
||||
logging.ERROR, logger="vllm.v1.kv_offload.cpu.gpu_worker"
|
||||
):
|
||||
worker.shutdown()
|
||||
|
||||
other = "load" if failing_handler == "store" else "store"
|
||||
getattr(worker, f"_{other}_handler").shutdown.assert_called_once_with()
|
||||
mmap_region.cleanup.assert_called_once_with()
|
||||
assert worker._mmap_region is None
|
||||
assert (
|
||||
f"Failed to shut down {failing_handler} offloading handler" in caplog_vllm.text
|
||||
)
|
||||
assert calls[-2:] == ["sync", "region"]
|
||||
|
||||
|
||||
def test_handler_shutdown_skips_transfers_after_event_sync_failure() -> None:
|
||||
handler = gpu_worker.SingleDirectionOffloadingHandler.__new__(
|
||||
gpu_worker.SingleDirectionOffloadingHandler
|
||||
)
|
||||
failed_event = MagicMock()
|
||||
failed_event.synchronize.side_effect = RuntimeError("device lost")
|
||||
skipped_event = MagicMock()
|
||||
handler._transfers = gpu_worker.deque(
|
||||
[
|
||||
MagicMock(end_event=failed_event),
|
||||
MagicMock(end_event=skipped_event),
|
||||
]
|
||||
)
|
||||
handler._transfer_events = {1: failed_event, 2: skipped_event}
|
||||
handler._stream_pool = [MagicMock()]
|
||||
handler._event_pool = [MagicMock()]
|
||||
handler._buffer_pool = [(MagicMock(), MagicMock(), MagicMock())]
|
||||
handler.src_tensors = [MagicMock()]
|
||||
handler.dst_tensors = [MagicMock()]
|
||||
|
||||
with pytest.raises(RuntimeError, match="device lost"):
|
||||
handler.shutdown()
|
||||
failed_event.synchronize.assert_called_once_with()
|
||||
skipped_event.synchronize.assert_not_called()
|
||||
assert not handler._transfers
|
||||
assert not handler._transfer_events
|
||||
assert not handler._stream_pool
|
||||
assert not handler._event_pool
|
||||
assert not handler._buffer_pool
|
||||
assert not handler.src_tensors
|
||||
assert not handler.dst_tensors
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_sync_fails", [False, True])
|
||||
def test_worker_syncs_before_cleanup_after_handler_failure(
|
||||
caplog_vllm, monkeypatch: pytest.MonkeyPatch, device_sync_fails: bool
|
||||
) -> None:
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
calls: list[str] = []
|
||||
store_handler = MagicMock()
|
||||
|
||||
def fail_store_shutdown() -> None:
|
||||
raise RuntimeError("device lost")
|
||||
|
||||
store_handler.shutdown.side_effect = fail_store_shutdown
|
||||
load_handler = MagicMock()
|
||||
|
||||
def record_device_sync() -> None:
|
||||
calls.append("sync")
|
||||
if device_sync_fails:
|
||||
raise RuntimeError("device lost")
|
||||
|
||||
def record_region_cleanup() -> None:
|
||||
calls.append("region")
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.cleanup.side_effect = record_region_cleanup
|
||||
worker._store_handler = store_handler
|
||||
worker._load_handler = load_handler
|
||||
worker._mmap_region = mmap_region
|
||||
|
||||
with caplog_vllm.at_level(
|
||||
logging.WARNING, logger="vllm.v1.kv_offload.cpu.gpu_worker"
|
||||
):
|
||||
worker.shutdown()
|
||||
|
||||
assert calls == ["sync", "region"]
|
||||
mmap_region.cleanup.assert_called_once_with()
|
||||
if device_sync_fails:
|
||||
assert "Device sync before mmap cleanup failed" in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@@ -229,8 +402,6 @@ def test_transfer(
|
||||
del src_view, dst_view, orig_dst_view, expected
|
||||
|
||||
worker.shutdown()
|
||||
if mmap_region:
|
||||
mmap_region.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
|
||||
@@ -248,7 +248,6 @@ class SingleDirectionOffloadingHandler:
|
||||
blocks_per_chunk: int,
|
||||
layer_refs_per_group: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
mmap_region: SharedOffloadRegion | None = None,
|
||||
canonical_layout: bool = False,
|
||||
):
|
||||
"""
|
||||
@@ -326,8 +325,6 @@ class SingleDirectionOffloadingHandler:
|
||||
self._scratch_bases_src = np.empty(num_scratch_blocks, dtype=np.uint64)
|
||||
self._scratch_bases_dst = np.empty(num_scratch_blocks, dtype=np.uint64)
|
||||
|
||||
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
|
||||
self._mmap_region = mmap_region
|
||||
# job_id -> event
|
||||
self._transfer_events: dict[int, torch.Event] = {}
|
||||
# queue of transfers (job_id, stream, event)
|
||||
@@ -711,18 +708,31 @@ class SingleDirectionOffloadingHandler:
|
||||
event.synchronize()
|
||||
|
||||
def shutdown(self) -> None:
|
||||
"""Drain this direction and release its transfer-side resources."""
|
||||
sync_error: Exception | None = None
|
||||
while self._transfers:
|
||||
transfer = self._transfers.popleft()
|
||||
transfer.end_event.synchronize()
|
||||
transfer = self._transfers[0]
|
||||
try:
|
||||
transfer.end_event.synchronize()
|
||||
except Exception as e:
|
||||
logger.exception(
|
||||
"Failed to synchronize transfer end event; "
|
||||
"skipping %d remaining transfers",
|
||||
len(self._transfers) - 1,
|
||||
)
|
||||
self._transfers.clear()
|
||||
sync_error = e
|
||||
break
|
||||
self._transfers.popleft()
|
||||
|
||||
self._transfer_events.clear()
|
||||
self._stream_pool.clear()
|
||||
self._event_pool.clear()
|
||||
self._buffer_pool.clear()
|
||||
self.src_tensors.clear()
|
||||
self.dst_tensors.clear()
|
||||
if self._mmap_region is not None:
|
||||
self._mmap_region.cleanup()
|
||||
self._mmap_region = None
|
||||
if sync_error is not None:
|
||||
raise sync_error
|
||||
|
||||
|
||||
class CPUOffloadingWorker(OffloadingWorker):
|
||||
@@ -742,6 +752,10 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
canonical_layout: bool = False,
|
||||
):
|
||||
assert not canonical_layout or mmap_region is not None
|
||||
# The caller owns mmap_region until this constructor returns. After a
|
||||
# successful construction, the worker is the sole owner and releases
|
||||
# it after both transfer directions have stopped.
|
||||
self._mmap_region = mmap_region
|
||||
pin_memory = PIN_MEMORY
|
||||
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
|
||||
if mmap_region is not None and pin_memory:
|
||||
@@ -794,7 +808,6 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
blocks_per_chunk=blocks_per_chunk,
|
||||
layer_refs_per_group=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=True,
|
||||
mmap_region=mmap_region,
|
||||
canonical_layout=canonical_layout,
|
||||
)
|
||||
|
||||
@@ -827,5 +840,28 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
self._load_handler.wait(job_ids)
|
||||
|
||||
def shutdown(self) -> None:
|
||||
self._store_handler.shutdown()
|
||||
self._load_handler.shutdown()
|
||||
handler_failed = False
|
||||
try:
|
||||
self._store_handler.shutdown()
|
||||
except Exception:
|
||||
logger.exception("Failed to shut down store offloading handler")
|
||||
handler_failed = True
|
||||
|
||||
try:
|
||||
self._load_handler.shutdown()
|
||||
except Exception:
|
||||
logger.exception("Failed to shut down load offloading handler")
|
||||
handler_failed = True
|
||||
|
||||
if self._mmap_region is not None:
|
||||
if handler_failed:
|
||||
try:
|
||||
torch.accelerator.synchronize()
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"Device sync before mmap cleanup failed; "
|
||||
"proceeding with cleanup anyway",
|
||||
exc_info=True,
|
||||
)
|
||||
self._mmap_region.cleanup()
|
||||
self._mmap_region = None
|
||||
|
||||
Reference in New Issue
Block a user