[Bugfix][KV Offload] Centralize shared mmap cleanup in CPU worker (#51622)

Signed-off-by: Alex <jihui.huang@daocloud.io>
This commit is contained in:
AlexHuang
2026-08-11 13:22:15 +03:00
committed by GitHub
parent ce07118669
commit 0fec3d652b
2 changed files with 220 additions and 13 deletions
+173 -2
View File
@@ -1,8 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import logging
import random
import time
import uuid
from unittest.mock import MagicMock
import pytest
import torch
@@ -46,6 +48,177 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
)
def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None:
"""Both directions drain before the worker releases its shared region."""
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
calls: list[str] = []
def record_store_shutdown() -> bool:
calls.append("store")
return True
def record_load_shutdown() -> bool:
calls.append("load")
return True
store_handler = MagicMock()
load_handler = MagicMock()
store_handler.shutdown.side_effect = record_store_shutdown
load_handler.shutdown.side_effect = record_load_shutdown
def record_region_cleanup(**_: bool) -> bool:
calls.append("region")
return True
mmap_region = MagicMock()
mmap_region.cleanup.side_effect = record_region_cleanup
worker._store_handler = store_handler
worker._load_handler = load_handler
worker._mmap_region = mmap_region
worker.shutdown()
assert calls == ["store", "load", "region"]
store_handler.shutdown.assert_called_once_with()
load_handler.shutdown.assert_called_once_with()
mmap_region.cleanup.assert_called_once_with()
assert worker._mmap_region is None
@pytest.mark.parametrize("failing_handler", ["store", "load"])
def test_worker_logs_handler_error_and_cleans_region(
caplog_vllm, monkeypatch: pytest.MonkeyPatch, failing_handler
) -> None:
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
calls: list[str] = []
store_handler = MagicMock()
load_handler = MagicMock()
if failing_handler == "store":
def fail_store_shutdown() -> bool:
calls.append("store")
raise RuntimeError("transfer did not drain")
store_handler.shutdown.side_effect = fail_store_shutdown
else:
def fail_load_shutdown() -> bool:
calls.append("load")
raise RuntimeError("transfer did not drain")
load_handler.shutdown.side_effect = fail_load_shutdown
def record_other_shutdown() -> bool:
calls.append("load" if failing_handler == "store" else "store")
return True
if failing_handler == "store":
load_handler.shutdown.side_effect = record_other_shutdown
else:
store_handler.shutdown.side_effect = record_other_shutdown
def record_device_sync() -> None:
calls.append("sync")
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
def record_region_cleanup() -> None:
calls.append("region")
mmap_region = MagicMock()
mmap_region.cleanup.side_effect = record_region_cleanup
worker._store_handler = store_handler
worker._load_handler = load_handler
worker._mmap_region = mmap_region
with caplog_vllm.at_level(
logging.ERROR, logger="vllm.v1.kv_offload.cpu.gpu_worker"
):
worker.shutdown()
other = "load" if failing_handler == "store" else "store"
getattr(worker, f"_{other}_handler").shutdown.assert_called_once_with()
mmap_region.cleanup.assert_called_once_with()
assert worker._mmap_region is None
assert (
f"Failed to shut down {failing_handler} offloading handler" in caplog_vllm.text
)
assert calls[-2:] == ["sync", "region"]
def test_handler_shutdown_skips_transfers_after_event_sync_failure() -> None:
handler = gpu_worker.SingleDirectionOffloadingHandler.__new__(
gpu_worker.SingleDirectionOffloadingHandler
)
failed_event = MagicMock()
failed_event.synchronize.side_effect = RuntimeError("device lost")
skipped_event = MagicMock()
handler._transfers = gpu_worker.deque(
[
MagicMock(end_event=failed_event),
MagicMock(end_event=skipped_event),
]
)
handler._transfer_events = {1: failed_event, 2: skipped_event}
handler._stream_pool = [MagicMock()]
handler._event_pool = [MagicMock()]
handler._buffer_pool = [(MagicMock(), MagicMock(), MagicMock())]
handler.src_tensors = [MagicMock()]
handler.dst_tensors = [MagicMock()]
with pytest.raises(RuntimeError, match="device lost"):
handler.shutdown()
failed_event.synchronize.assert_called_once_with()
skipped_event.synchronize.assert_not_called()
assert not handler._transfers
assert not handler._transfer_events
assert not handler._stream_pool
assert not handler._event_pool
assert not handler._buffer_pool
assert not handler.src_tensors
assert not handler.dst_tensors
@pytest.mark.parametrize("device_sync_fails", [False, True])
def test_worker_syncs_before_cleanup_after_handler_failure(
caplog_vllm, monkeypatch: pytest.MonkeyPatch, device_sync_fails: bool
) -> None:
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
calls: list[str] = []
store_handler = MagicMock()
def fail_store_shutdown() -> None:
raise RuntimeError("device lost")
store_handler.shutdown.side_effect = fail_store_shutdown
load_handler = MagicMock()
def record_device_sync() -> None:
calls.append("sync")
if device_sync_fails:
raise RuntimeError("device lost")
def record_region_cleanup() -> None:
calls.append("region")
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
mmap_region = MagicMock()
mmap_region.cleanup.side_effect = record_region_cleanup
worker._store_handler = store_handler
worker._load_handler = load_handler
worker._mmap_region = mmap_region
with caplog_vllm.at_level(
logging.WARNING, logger="vllm.v1.kv_offload.cpu.gpu_worker"
):
worker.shutdown()
assert calls == ["sync", "region"]
mmap_region.cleanup.assert_called_once_with()
if device_sync_fails:
assert "Device sync before mmap cleanup failed" in caplog_vllm.text
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
@@ -229,8 +402,6 @@ def test_transfer(
del src_view, dst_view, orig_dst_view, expected
worker.shutdown()
if mmap_region:
mmap_region.cleanup()
@pytest.mark.parametrize("gpu_to_cpu", [True, False])