diff --git a/tests/v1/kv_offload/cpu/test_gpu_worker.py b/tests/v1/kv_offload/cpu/test_gpu_worker.py index 694c33712e..9a90dedb3d 100644 --- a/tests/v1/kv_offload/cpu/test_gpu_worker.py +++ b/tests/v1/kv_offload/cpu/test_gpu_worker.py @@ -1,8 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import logging import random import time import uuid +from unittest.mock import MagicMock import pytest import torch @@ -46,6 +48,177 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: ) +def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None: + """Both directions drain before the worker releases its shared region.""" + worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker) + calls: list[str] = [] + + def record_store_shutdown() -> bool: + calls.append("store") + return True + + def record_load_shutdown() -> bool: + calls.append("load") + return True + + store_handler = MagicMock() + load_handler = MagicMock() + store_handler.shutdown.side_effect = record_store_shutdown + load_handler.shutdown.side_effect = record_load_shutdown + + def record_region_cleanup(**_: bool) -> bool: + calls.append("region") + return True + + mmap_region = MagicMock() + mmap_region.cleanup.side_effect = record_region_cleanup + worker._store_handler = store_handler + worker._load_handler = load_handler + worker._mmap_region = mmap_region + + worker.shutdown() + + assert calls == ["store", "load", "region"] + store_handler.shutdown.assert_called_once_with() + load_handler.shutdown.assert_called_once_with() + mmap_region.cleanup.assert_called_once_with() + assert worker._mmap_region is None + + +@pytest.mark.parametrize("failing_handler", ["store", "load"]) +def test_worker_logs_handler_error_and_cleans_region( + caplog_vllm, monkeypatch: pytest.MonkeyPatch, failing_handler +) -> None: + worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker) + calls: list[str] = [] + store_handler = MagicMock() + load_handler = MagicMock() + if failing_handler == "store": + + def fail_store_shutdown() -> bool: + calls.append("store") + raise RuntimeError("transfer did not drain") + + store_handler.shutdown.side_effect = fail_store_shutdown + else: + + def fail_load_shutdown() -> bool: + calls.append("load") + raise RuntimeError("transfer did not drain") + + load_handler.shutdown.side_effect = fail_load_shutdown + + def record_other_shutdown() -> bool: + calls.append("load" if failing_handler == "store" else "store") + return True + + if failing_handler == "store": + load_handler.shutdown.side_effect = record_other_shutdown + else: + store_handler.shutdown.side_effect = record_other_shutdown + + def record_device_sync() -> None: + calls.append("sync") + + monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync) + + def record_region_cleanup() -> None: + calls.append("region") + + mmap_region = MagicMock() + mmap_region.cleanup.side_effect = record_region_cleanup + worker._store_handler = store_handler + worker._load_handler = load_handler + worker._mmap_region = mmap_region + + with caplog_vllm.at_level( + logging.ERROR, logger="vllm.v1.kv_offload.cpu.gpu_worker" + ): + worker.shutdown() + + other = "load" if failing_handler == "store" else "store" + getattr(worker, f"_{other}_handler").shutdown.assert_called_once_with() + mmap_region.cleanup.assert_called_once_with() + assert worker._mmap_region is None + assert ( + f"Failed to shut down {failing_handler} offloading handler" in caplog_vllm.text + ) + assert calls[-2:] == ["sync", "region"] + + +def test_handler_shutdown_skips_transfers_after_event_sync_failure() -> None: + handler = gpu_worker.SingleDirectionOffloadingHandler.__new__( + gpu_worker.SingleDirectionOffloadingHandler + ) + failed_event = MagicMock() + failed_event.synchronize.side_effect = RuntimeError("device lost") + skipped_event = MagicMock() + handler._transfers = gpu_worker.deque( + [ + MagicMock(end_event=failed_event), + MagicMock(end_event=skipped_event), + ] + ) + handler._transfer_events = {1: failed_event, 2: skipped_event} + handler._stream_pool = [MagicMock()] + handler._event_pool = [MagicMock()] + handler._buffer_pool = [(MagicMock(), MagicMock(), MagicMock())] + handler.src_tensors = [MagicMock()] + handler.dst_tensors = [MagicMock()] + + with pytest.raises(RuntimeError, match="device lost"): + handler.shutdown() + failed_event.synchronize.assert_called_once_with() + skipped_event.synchronize.assert_not_called() + assert not handler._transfers + assert not handler._transfer_events + assert not handler._stream_pool + assert not handler._event_pool + assert not handler._buffer_pool + assert not handler.src_tensors + assert not handler.dst_tensors + + +@pytest.mark.parametrize("device_sync_fails", [False, True]) +def test_worker_syncs_before_cleanup_after_handler_failure( + caplog_vllm, monkeypatch: pytest.MonkeyPatch, device_sync_fails: bool +) -> None: + worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker) + calls: list[str] = [] + store_handler = MagicMock() + + def fail_store_shutdown() -> None: + raise RuntimeError("device lost") + + store_handler.shutdown.side_effect = fail_store_shutdown + load_handler = MagicMock() + + def record_device_sync() -> None: + calls.append("sync") + if device_sync_fails: + raise RuntimeError("device lost") + + def record_region_cleanup() -> None: + calls.append("region") + + monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync) + mmap_region = MagicMock() + mmap_region.cleanup.side_effect = record_region_cleanup + worker._store_handler = store_handler + worker._load_handler = load_handler + worker._mmap_region = mmap_region + + with caplog_vllm.at_level( + logging.WARNING, logger="vllm.v1.kv_offload.cpu.gpu_worker" + ): + worker.shutdown() + + assert calls == ["sync", "region"] + mmap_region.cleanup.assert_called_once_with() + if device_sync_fails: + assert "Device sync before mmap cleanup failed" in caplog_vllm.text + + @pytest.mark.parametrize("gpu_to_cpu", [True, False]) @pytest.mark.parametrize("num_mappings", NUM_MAPPINGS) @pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES) @@ -229,8 +402,6 @@ def test_transfer( del src_view, dst_view, orig_dst_view, expected worker.shutdown() - if mmap_region: - mmap_region.cleanup() @pytest.mark.parametrize("gpu_to_cpu", [True, False]) diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index c7d75b636c..ab4fedcbe0 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -248,7 +248,6 @@ class SingleDirectionOffloadingHandler: blocks_per_chunk: int, layer_refs_per_group: list[list[CanonicalKVCacheRef]], gpu_to_cpu: bool, - mmap_region: SharedOffloadRegion | None = None, canonical_layout: bool = False, ): """ @@ -326,8 +325,6 @@ class SingleDirectionOffloadingHandler: self._scratch_bases_src = np.empty(num_scratch_blocks, dtype=np.uint64) self._scratch_bases_dst = np.empty(num_scratch_blocks, dtype=np.uint64) - # mmap_region to clean up on shutdown (gpu_to_cpu handler owns it) - self._mmap_region = mmap_region # job_id -> event self._transfer_events: dict[int, torch.Event] = {} # queue of transfers (job_id, stream, event) @@ -711,18 +708,31 @@ class SingleDirectionOffloadingHandler: event.synchronize() def shutdown(self) -> None: + """Drain this direction and release its transfer-side resources.""" + sync_error: Exception | None = None while self._transfers: - transfer = self._transfers.popleft() - transfer.end_event.synchronize() + transfer = self._transfers[0] + try: + transfer.end_event.synchronize() + except Exception as e: + logger.exception( + "Failed to synchronize transfer end event; " + "skipping %d remaining transfers", + len(self._transfers) - 1, + ) + self._transfers.clear() + sync_error = e + break + self._transfers.popleft() + self._transfer_events.clear() self._stream_pool.clear() self._event_pool.clear() self._buffer_pool.clear() self.src_tensors.clear() self.dst_tensors.clear() - if self._mmap_region is not None: - self._mmap_region.cleanup() - self._mmap_region = None + if sync_error is not None: + raise sync_error class CPUOffloadingWorker(OffloadingWorker): @@ -742,6 +752,10 @@ class CPUOffloadingWorker(OffloadingWorker): canonical_layout: bool = False, ): assert not canonical_layout or mmap_region is not None + # The caller owns mmap_region until this constructor returns. After a + # successful construction, the worker is the sole owner and releases + # it after both transfer directions have stopped. + self._mmap_region = mmap_region pin_memory = PIN_MEMORY logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors)) if mmap_region is not None and pin_memory: @@ -794,7 +808,6 @@ class CPUOffloadingWorker(OffloadingWorker): blocks_per_chunk=blocks_per_chunk, layer_refs_per_group=kv_caches.group_data_refs, gpu_to_cpu=True, - mmap_region=mmap_region, canonical_layout=canonical_layout, ) @@ -827,5 +840,28 @@ class CPUOffloadingWorker(OffloadingWorker): self._load_handler.wait(job_ids) def shutdown(self) -> None: - self._store_handler.shutdown() - self._load_handler.shutdown() + handler_failed = False + try: + self._store_handler.shutdown() + except Exception: + logger.exception("Failed to shut down store offloading handler") + handler_failed = True + + try: + self._load_handler.shutdown() + except Exception: + logger.exception("Failed to shut down load offloading handler") + handler_failed = True + + if self._mmap_region is not None: + if handler_failed: + try: + torch.accelerator.synchronize() + except Exception: + logger.warning( + "Device sync before mmap cleanup failed; " + "proceeding with cleanup anyway", + exc_info=True, + ) + self._mmap_region.cleanup() + self._mmap_region = None