[Bugfix][KV Offload] Retain offload mappings after unsafe KVCR startup failure (#57500)
Signed-off-by: Kapil Arya <kapila@nvidia.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
@@ -1060,6 +1060,22 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_cleanup_keeps_region_mapped_while_buffers_are_exported(iid):
|
||||
"""A native user's buffer export must block unmapping -- unmapping under it
|
||||
would let the NIC write into reused address space. The pathname still goes."""
|
||||
region = _make_region(iid)
|
||||
export = ctypes.c_char.from_buffer(region.mmap_obj)
|
||||
try:
|
||||
region.cleanup()
|
||||
assert region.mmap_obj is not None and not region.mmap_obj.closed
|
||||
assert not os.path.exists(region.mmap_path), "pathname must still go"
|
||||
assert region.mmap_obj[0:1] == b"\x00", "mapping must stay readable"
|
||||
finally:
|
||||
del export
|
||||
region.mmap_obj.close()
|
||||
_cleanup_file(region.mmap_path)
|
||||
|
||||
|
||||
def test_backing_file_unlinked_after_barrier(iid):
|
||||
"""The file must exist until the barrier releases (late joiners need the
|
||||
name) and be gone right after, so no exit path can leak it."""
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import mmap
|
||||
from collections.abc import Collection, Iterable, Mapping
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import Mock
|
||||
|
||||
import msgspec
|
||||
import numpy as np
|
||||
@@ -18,6 +20,7 @@ from kvcr.types import (
|
||||
BlockKey,
|
||||
CacheTier,
|
||||
InventoryEvent,
|
||||
KVCRStartupError,
|
||||
MemoryRef,
|
||||
OpEntryResult,
|
||||
OpEntryStatus,
|
||||
@@ -172,6 +175,8 @@ def _make_tier(
|
||||
monkeypatch,
|
||||
kvcr: RecordingKVCR,
|
||||
*,
|
||||
startup_error: BaseException | None = None,
|
||||
primary_kv_view: memoryview | None = None,
|
||||
enable_telemetry: bool = False,
|
||||
secondary_g2_slots: int = 0,
|
||||
kvcr_service_socket_path: str | None = None,
|
||||
@@ -195,6 +200,8 @@ def _make_tier(
|
||||
kvcr.constructor_bindings = bindings
|
||||
kvcr.framework_control = bindings.framework_control
|
||||
kvcr.inventory_sink = bindings.inventory_sink
|
||||
if startup_error is not None:
|
||||
raise startup_error
|
||||
return kvcr
|
||||
|
||||
monkeypatch.setattr(kvcr_manager, "KVCR", make_kvcr)
|
||||
@@ -229,12 +236,61 @@ def _make_tier(
|
||||
self_describing_kv_events=self_describing_kv_events,
|
||||
),
|
||||
),
|
||||
primary_kv_view=memoryview(np.zeros((4, 16), dtype=np.int8)),
|
||||
primary_kv_view=(
|
||||
primary_kv_view
|
||||
if primary_kv_view is not None
|
||||
else memoryview(np.zeros((4, 16), dtype=np.int8))
|
||||
),
|
||||
)
|
||||
assert isinstance(tier, KVCRSecondaryTierManager)
|
||||
return tier
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"error_type", [RuntimeError, KVCRStartupError, KeyboardInterrupt]
|
||||
)
|
||||
@pytest.mark.parametrize("secondary_g2_slots", [0, 1])
|
||||
def test_startup_failure_cleanup(monkeypatch, error_type, secondary_g2_slots):
|
||||
"""Unsafe failures must hold a buffer export so nothing can unmap memory
|
||||
native work may still write into. Ordinary failures release everything."""
|
||||
retained: list[object] = []
|
||||
monkeypatch.setattr(kvcr_manager, "_RETAINED_NATIVE_BUFFERS", retained)
|
||||
close = Mock()
|
||||
monkeypatch.setattr(_StubControlChannel, "close", close)
|
||||
unsafe = error_type is not RuntimeError
|
||||
|
||||
mapping = mmap.mmap(-1, mmap.PAGESIZE)
|
||||
primary_kv_view = memoryview(mapping)
|
||||
local_mapping = mmap.mmap(-1, mmap.PAGESIZE) if secondary_g2_slots else None
|
||||
monkeypatch.setattr(
|
||||
kvcr_manager, "mmap", SimpleNamespace(mmap=lambda *_: local_mapping)
|
||||
)
|
||||
|
||||
with pytest.raises(error_type, match="startup failed"):
|
||||
_make_tier(
|
||||
monkeypatch,
|
||||
RecordingKVCR(),
|
||||
startup_error=error_type("startup failed"),
|
||||
primary_kv_view=primary_kv_view,
|
||||
secondary_g2_slots=secondary_g2_slots,
|
||||
)
|
||||
|
||||
assert close.call_count == (0 if unsafe else 1)
|
||||
# Drop the test's own export, leaving only whatever the manager retained:
|
||||
# the mapping is unmappable afterwards only if that retention covers it.
|
||||
primary_kv_view.release()
|
||||
mappings = [mapping] if local_mapping is None else [mapping, local_mapping]
|
||||
if local_mapping is not None:
|
||||
assert local_mapping.closed is not unsafe
|
||||
for region in mappings:
|
||||
if unsafe:
|
||||
with pytest.raises(BufferError):
|
||||
region.close()
|
||||
retained.clear()
|
||||
for region in mappings:
|
||||
region.close()
|
||||
|
||||
|
||||
def test_kvcr_tier_configures_service_for_local_dp_rank(monkeypatch):
|
||||
"""Keep the control endpoint and guard pool aligned to the local DP rank."""
|
||||
kvcr = RecordingKVCR()
|
||||
|
||||
@@ -368,11 +368,22 @@ class SharedOffloadRegion:
|
||||
self._views.clear()
|
||||
self._base = None
|
||||
if self.mmap_obj:
|
||||
retained = False
|
||||
try:
|
||||
self.mmap_obj.close()
|
||||
except BufferError:
|
||||
# A native user still exports this mapping, so unmapping would
|
||||
# let the NIC write into reused address space. Keep it mapped;
|
||||
# dropping the fd and the pathname below is still safe.
|
||||
retained = True
|
||||
logger.warning(
|
||||
"Keeping %s mapped: buffers are still exported to native users",
|
||||
self.mmap_path,
|
||||
)
|
||||
except Exception:
|
||||
logger.warning("Failed to close mmap_obj", exc_info=True)
|
||||
self.mmap_obj = None
|
||||
if not retained:
|
||||
self.mmap_obj = None
|
||||
if self.fd is not None:
|
||||
try:
|
||||
os.close(self.fd)
|
||||
|
||||
@@ -42,6 +42,7 @@ from kvcr.types import (
|
||||
BlockKey,
|
||||
CacheTier,
|
||||
InventoryEvent,
|
||||
KVCRStartupError,
|
||||
MemoryRef,
|
||||
OpHandle,
|
||||
PinRequestId,
|
||||
@@ -262,6 +263,12 @@ class _VllmKeyAdapter:
|
||||
_JobState = tuple[int, int, bool, set[OffloadKey] | None]
|
||||
|
||||
|
||||
# Held for the process lifetime after a startup failure that could not stop
|
||||
# native work: an export blocks close() and keeps the mapping alive against GC,
|
||||
# so nothing can unmap memory NIXL may still be writing into.
|
||||
_RETAINED_NATIVE_BUFFERS: list[object] = []
|
||||
|
||||
|
||||
class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
"""Secondary tier wrapper around the KVCR KV P2P API."""
|
||||
|
||||
@@ -447,7 +454,17 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
|
||||
),
|
||||
guard_config,
|
||||
)
|
||||
except BaseException:
|
||||
except BaseException as exc:
|
||||
if not isinstance(exc, Exception) or isinstance(exc, KVCRStartupError):
|
||||
# Native work may still touch these.
|
||||
_RETAINED_NATIVE_BUFFERS.append(
|
||||
ctypes.c_char.from_buffer(primary_kv_view)
|
||||
)
|
||||
if local_mapping is not None:
|
||||
_RETAINED_NATIVE_BUFFERS.append(
|
||||
ctypes.c_char.from_buffer(local_mapping)
|
||||
)
|
||||
raise
|
||||
control.close()
|
||||
if local_mapping is not None:
|
||||
local_mapping.close()
|
||||
|
||||
Reference in New Issue
Block a user