[Bugfix][KV Offload] Centralize shared mmap cleanup in CPU worker (#51622)
Signed-off-by: Alex <jihui.huang@daocloud.io>
This commit is contained in:
@@ -1,8 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
import uuid
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
@@ -46,6 +48,177 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
)
|
||||
|
||||
|
||||
def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None:
|
||||
"""Both directions drain before the worker releases its shared region."""
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
calls: list[str] = []
|
||||
|
||||
def record_store_shutdown() -> bool:
|
||||
calls.append("store")
|
||||
return True
|
||||
|
||||
def record_load_shutdown() -> bool:
|
||||
calls.append("load")
|
||||
return True
|
||||
|
||||
store_handler = MagicMock()
|
||||
load_handler = MagicMock()
|
||||
store_handler.shutdown.side_effect = record_store_shutdown
|
||||
load_handler.shutdown.side_effect = record_load_shutdown
|
||||
|
||||
def record_region_cleanup(**_: bool) -> bool:
|
||||
calls.append("region")
|
||||
return True
|
||||
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.cleanup.side_effect = record_region_cleanup
|
||||
worker._store_handler = store_handler
|
||||
worker._load_handler = load_handler
|
||||
worker._mmap_region = mmap_region
|
||||
|
||||
worker.shutdown()
|
||||
|
||||
assert calls == ["store", "load", "region"]
|
||||
store_handler.shutdown.assert_called_once_with()
|
||||
load_handler.shutdown.assert_called_once_with()
|
||||
mmap_region.cleanup.assert_called_once_with()
|
||||
assert worker._mmap_region is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("failing_handler", ["store", "load"])
|
||||
def test_worker_logs_handler_error_and_cleans_region(
|
||||
caplog_vllm, monkeypatch: pytest.MonkeyPatch, failing_handler
|
||||
) -> None:
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
calls: list[str] = []
|
||||
store_handler = MagicMock()
|
||||
load_handler = MagicMock()
|
||||
if failing_handler == "store":
|
||||
|
||||
def fail_store_shutdown() -> bool:
|
||||
calls.append("store")
|
||||
raise RuntimeError("transfer did not drain")
|
||||
|
||||
store_handler.shutdown.side_effect = fail_store_shutdown
|
||||
else:
|
||||
|
||||
def fail_load_shutdown() -> bool:
|
||||
calls.append("load")
|
||||
raise RuntimeError("transfer did not drain")
|
||||
|
||||
load_handler.shutdown.side_effect = fail_load_shutdown
|
||||
|
||||
def record_other_shutdown() -> bool:
|
||||
calls.append("load" if failing_handler == "store" else "store")
|
||||
return True
|
||||
|
||||
if failing_handler == "store":
|
||||
load_handler.shutdown.side_effect = record_other_shutdown
|
||||
else:
|
||||
store_handler.shutdown.side_effect = record_other_shutdown
|
||||
|
||||
def record_device_sync() -> None:
|
||||
calls.append("sync")
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
|
||||
|
||||
def record_region_cleanup() -> None:
|
||||
calls.append("region")
|
||||
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.cleanup.side_effect = record_region_cleanup
|
||||
worker._store_handler = store_handler
|
||||
worker._load_handler = load_handler
|
||||
worker._mmap_region = mmap_region
|
||||
|
||||
with caplog_vllm.at_level(
|
||||
logging.ERROR, logger="vllm.v1.kv_offload.cpu.gpu_worker"
|
||||
):
|
||||
worker.shutdown()
|
||||
|
||||
other = "load" if failing_handler == "store" else "store"
|
||||
getattr(worker, f"_{other}_handler").shutdown.assert_called_once_with()
|
||||
mmap_region.cleanup.assert_called_once_with()
|
||||
assert worker._mmap_region is None
|
||||
assert (
|
||||
f"Failed to shut down {failing_handler} offloading handler" in caplog_vllm.text
|
||||
)
|
||||
assert calls[-2:] == ["sync", "region"]
|
||||
|
||||
|
||||
def test_handler_shutdown_skips_transfers_after_event_sync_failure() -> None:
|
||||
handler = gpu_worker.SingleDirectionOffloadingHandler.__new__(
|
||||
gpu_worker.SingleDirectionOffloadingHandler
|
||||
)
|
||||
failed_event = MagicMock()
|
||||
failed_event.synchronize.side_effect = RuntimeError("device lost")
|
||||
skipped_event = MagicMock()
|
||||
handler._transfers = gpu_worker.deque(
|
||||
[
|
||||
MagicMock(end_event=failed_event),
|
||||
MagicMock(end_event=skipped_event),
|
||||
]
|
||||
)
|
||||
handler._transfer_events = {1: failed_event, 2: skipped_event}
|
||||
handler._stream_pool = [MagicMock()]
|
||||
handler._event_pool = [MagicMock()]
|
||||
handler._buffer_pool = [(MagicMock(), MagicMock(), MagicMock())]
|
||||
handler.src_tensors = [MagicMock()]
|
||||
handler.dst_tensors = [MagicMock()]
|
||||
|
||||
with pytest.raises(RuntimeError, match="device lost"):
|
||||
handler.shutdown()
|
||||
failed_event.synchronize.assert_called_once_with()
|
||||
skipped_event.synchronize.assert_not_called()
|
||||
assert not handler._transfers
|
||||
assert not handler._transfer_events
|
||||
assert not handler._stream_pool
|
||||
assert not handler._event_pool
|
||||
assert not handler._buffer_pool
|
||||
assert not handler.src_tensors
|
||||
assert not handler.dst_tensors
|
||||
|
||||
|
||||
@pytest.mark.parametrize("device_sync_fails", [False, True])
|
||||
def test_worker_syncs_before_cleanup_after_handler_failure(
|
||||
caplog_vllm, monkeypatch: pytest.MonkeyPatch, device_sync_fails: bool
|
||||
) -> None:
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
calls: list[str] = []
|
||||
store_handler = MagicMock()
|
||||
|
||||
def fail_store_shutdown() -> None:
|
||||
raise RuntimeError("device lost")
|
||||
|
||||
store_handler.shutdown.side_effect = fail_store_shutdown
|
||||
load_handler = MagicMock()
|
||||
|
||||
def record_device_sync() -> None:
|
||||
calls.append("sync")
|
||||
if device_sync_fails:
|
||||
raise RuntimeError("device lost")
|
||||
|
||||
def record_region_cleanup() -> None:
|
||||
calls.append("region")
|
||||
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", record_device_sync)
|
||||
mmap_region = MagicMock()
|
||||
mmap_region.cleanup.side_effect = record_region_cleanup
|
||||
worker._store_handler = store_handler
|
||||
worker._load_handler = load_handler
|
||||
worker._mmap_region = mmap_region
|
||||
|
||||
with caplog_vllm.at_level(
|
||||
logging.WARNING, logger="vllm.v1.kv_offload.cpu.gpu_worker"
|
||||
):
|
||||
worker.shutdown()
|
||||
|
||||
assert calls == ["sync", "region"]
|
||||
mmap_region.cleanup.assert_called_once_with()
|
||||
if device_sync_fails:
|
||||
assert "Device sync before mmap cleanup failed" in caplog_vllm.text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
@pytest.mark.parametrize("num_mappings", NUM_MAPPINGS)
|
||||
@pytest.mark.parametrize("gpu_page_size_bytes", GPU_PAGE_SIZES)
|
||||
@@ -229,8 +402,6 @@ def test_transfer(
|
||||
del src_view, dst_view, orig_dst_view, expected
|
||||
|
||||
worker.shutdown()
|
||||
if mmap_region:
|
||||
mmap_region.cleanup()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("gpu_to_cpu", [True, False])
|
||||
|
||||
Reference in New Issue
Block a user