[Bugfix][KV Offload] Centralize shared mmap cleanup in CPU worker (#51622)

Signed-off-by: Alex <jihui.huang@daocloud.io>
This commit is contained in:
AlexHuang
2026-08-11 13:22:15 +03:00
committed by GitHub
parent ce07118669
commit 0fec3d652b
2 changed files with 220 additions and 13 deletions
+173 -2
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import random
import time
import uuid
from unittest.mock import MagicMock
import pytest
import torch
@@ -46,6 +48,177 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
)
def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None:
"""Both directions drain before the worker releases its shared region."""
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
calls: list[str] = []
def record_store_shutdown() -> bool:
calls.append("store")
return True
def record_load_shutdown() -> bool:
calls.append("load")
return True
store_handler = MagicMock()
load_handler = MagicMock()
store_handler.shutdown.side_effect = record_store_shutdown
load_handler.shutdown.side_effect = record_load_shutdown
def record_region_cleanup(**_: bool) -> bool:
calls.append("region")
return True
mmap_region = MagicMock()
mmap_region.cleanup.side_effect = record_region_cleanup
worker._store_handler = store_handler
worker._load_handler = load_handler
worker._mmap_region = mmap_region
worker.shutdown()
assert calls == ["store", "load", "region"]
store_handler.shutdown.assert_called_once_with()
load_handler.shutdown.assert_called_once_with()
mmap_region.cleanup.assert_called_once_with()
assert worker._mmap_region is None
@pytest.mark.parametrize("failing_handler", ["store", "load"])
def test_worker_logs_handler_error_and_cleans_region(
caplog_vllm, monkeypatch: pytest.MonkeyPatch, failing_handler
) -> None:
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
calls: list[str] = []
store_handler = MagicMock()
load_handler = MagicMock()
if failing_handler == "store":
def fail_store_shutdown() -> bool:
calls.append("store")
raise RuntimeError("transfer did not drain")
store_handler.shutdown.side_effect = fail_store_shutdown
else:
def fail_load_shutdown() -> bool:
calls.append("load")
raise RuntimeError("transfer did not drain")
load_handler.shutdown.side_effect = fail_load_shutdown
def record_other_shutdown() -> bool:
calls.append("load" if failing_handler == "store" else "store")
return True
if failing_handler == "store":
load_handler.shutdown.side_effect = record_other_shutdown
else:
store_handler.shutdown.side_effect = record_other_shutdown
def record_device_sync() -> None:
calls.append("sync")
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
def record_region_cleanup() -> None:
calls.append("region")
mmap_region = MagicMock()
mmap_region.cleanup.side_effect = record_region_cleanup
worker._store_handler = store_handler
worker._load_handler = load_handler
worker._mmap_region = mmap_region
with caplog_vllm.at_level(
logging.ERROR, logger="vllm.v1.kv_offload.cpu.gpu_worker"
):
worker.shutdown()
other = "load" if failing_handler == "store" else "store"
getattr(worker, f"_{other}_handler").shutdown.assert_called_once_with()
mmap_region.cleanup.assert_called_once_with()
assert worker._mmap_region is None
assert (
f"Failed to shut down {failing_handler} offloading handler" in caplog_vllm.text
)
assert calls[-2:] == ["sync", "region"]
def test_handler_shutdown_skips_transfers_after_event_sync_failure() -> None:
handler = gpu_worker.SingleDirectionOffloadingHandler.__new__(
gpu_worker.SingleDirectionOffloadingHandler
)
failed_event = MagicMock()
failed_event.synchronize.side_effect = RuntimeError("device lost")
skipped_event = MagicMock()
handler._transfers = gpu_worker.deque(
[
MagicMock(end_event=failed_event),
MagicMock(end_event=skipped_event),
]
)
handler._transfer_events = {1: failed_event, 2: skipped_event}
handler._stream_pool = [MagicMock()]
handler._event_pool = [MagicMock()]
handler._buffer_pool = [(MagicMock(), MagicMock(), MagicMock())]
handler.src_tensors = [MagicMock()]
handler.dst_tensors = [MagicMock()]
with pytest.raises(RuntimeError, match="device lost"):
handler.shutdown()
failed_event.synchronize.assert_called_once_with()
skipped_event.synchronize.assert_not_called()
assert not handler._transfers
assert not handler._transfer_events
assert not handler._stream_pool
assert not handler._event_pool
assert not handler._buffer_pool
assert not handler.src_tensors
assert not handler.dst_tensors
@pytest.mark.parametrize("device_sync_fails", [False, True])
def test_worker_syncs_before_cleanup_after_handler_failure(
caplog_vllm, monkeypatch: pytest.MonkeyPatch, device_sync_fails: bool
) -> None:
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
calls: list[str] = []
store_handler = MagicMock()
def fail_store_shutdown() -> None:
raise RuntimeError("device lost")
store_handler.shutdown.side_effect = fail_store_shutdown
load_handler = MagicMock()
def record_device_sync() -> None:
calls.append("sync")
if device_sync_fails:
raise RuntimeError("device lost")
def record_region_cleanup() -> None:
calls.append("region")
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
mmap_region = MagicMock()
mmap_region.cleanup.side_effect = record_region_cleanup
worker._store_handler = store_handler
worker._load_handler = load_handler
worker._mmap_region = mmap_region
with caplog_vllm.at_level(
logging.WARNING, logger="vllm.v1.kv_offload.cpu.gpu_worker"
):
worker.shutdown()
assert calls == ["sync", "region"]
mmap_region.cleanup.assert_called_once_with()
if device_sync_fails:
assert "Device sync before mmap cleanup failed" in caplog_vllm.text
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@@ -229,8 +402,6 @@ def test_transfer(
del src_view, dst_view, orig_dst_view, expected
worker.shutdown()
if mmap_region:
mmap_region.cleanup()
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
+47 -11
View File
@@ -248,7 +248,6 @@ class SingleDirectionOffloadingHandler:
blocks_per_chunk: int,
layer_refs_per_group: list[list[CanonicalKVCacheRef]],
gpu_to_cpu: bool,
mmap_region: SharedOffloadRegion | None = None,
canonical_layout: bool = False,
):
"""
@@ -326,8 +325,6 @@ class SingleDirectionOffloadingHandler:
self._scratch_bases_src = np.empty(num_scratch_blocks, dtype=np.uint64)
self._scratch_bases_dst = np.empty(num_scratch_blocks, dtype=np.uint64)
# mmap_region to clean up on shutdown (gpu_to_cpu handler owns it)
self._mmap_region = mmap_region
# job_id -> event
self._transfer_events: dict[int, torch.Event] = {}
# queue of transfers (job_id, stream, event)
@@ -711,18 +708,31 @@ class SingleDirectionOffloadingHandler:
event.synchronize()
def shutdown(self) -> None:
"""Drain this direction and release its transfer-side resources."""
sync_error: Exception | None = None
while self._transfers:
transfer = self._transfers.popleft()
transfer.end_event.synchronize()
transfer = self._transfers[0]
try:
transfer.end_event.synchronize()
except Exception as e:
logger.exception(
"Failed to synchronize transfer end event; "
"skipping %d remaining transfers",
len(self._transfers) - 1,
)
self._transfers.clear()
sync_error = e
break
self._transfers.popleft()
self._transfer_events.clear()
self._stream_pool.clear()
self._event_pool.clear()
self._buffer_pool.clear()
self.src_tensors.clear()
self.dst_tensors.clear()
if self._mmap_region is not None:
self._mmap_region.cleanup()
self._mmap_region = None
if sync_error is not None:
raise sync_error
class CPUOffloadingWorker(OffloadingWorker):
@@ -742,6 +752,10 @@ class CPUOffloadingWorker(OffloadingWorker):
canonical_layout: bool = False,
):
assert not canonical_layout or mmap_region is not None
# The caller owns mmap_region until this constructor returns. After a
# successful construction, the worker is the sole owner and releases
# it after both transfer directions have stopped.
self._mmap_region = mmap_region
pin_memory = PIN_MEMORY
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
if mmap_region is not None and pin_memory:
@@ -794,7 +808,6 @@ class CPUOffloadingWorker(OffloadingWorker):
blocks_per_chunk=blocks_per_chunk,
layer_refs_per_group=kv_caches.group_data_refs,
gpu_to_cpu=True,
mmap_region=mmap_region,
canonical_layout=canonical_layout,
)
@@ -827,5 +840,28 @@ class CPUOffloadingWorker(OffloadingWorker):
self._load_handler.wait(job_ids)
def shutdown(self) -> None:
self._store_handler.shutdown()
self._load_handler.shutdown()
handler_failed = False
try:
self._store_handler.shutdown()
except Exception:
logger.exception("Failed to shut down store offloading handler")
handler_failed = True
try:
self._load_handler.shutdown()
except Exception:
logger.exception("Failed to shut down load offloading handler")
handler_failed = True
if self._mmap_region is not None:
if handler_failed:
try:
torch.accelerator.synchronize()
except Exception:
logger.warning(
"Device sync before mmap cleanup failed; "
"proceeding with cleanup anyway",
exc_info=True,
)
self._mmap_region.cleanup()
self._mmap_region = None