[Bugfix][KV Offload] Retain offload mappings after unsafe KVCR startup failure (#57500)

Signed-off-by: Kapil Arya <kapila@nvidia.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Kapil Arya
2026-10-14 15:43:04 +03:00
committed by GitHub
co-authored by Or Ozeri
parent 775fefa804
commit a68dbbad05
4 changed files with 103 additions and 3 deletions
@@ -1060,6 +1060,22 @@ def test_ftruncate_failure_cleans_up_creator(monkeypatch):
# ---------------------------------------------------------------------------
def test_cleanup_keeps_region_mapped_while_buffers_are_exported(iid):
"""A native user's buffer export must block unmapping -- unmapping under it
would let the NIC write into reused address space. The pathname still goes."""
region = _make_region(iid)
export = ctypes.c_char.from_buffer(region.mmap_obj)
try:
region.cleanup()
assert region.mmap_obj is not None and not region.mmap_obj.closed
assert not os.path.exists(region.mmap_path), "pathname must still go"
assert region.mmap_obj[0:1] == b"\x00", "mapping must stay readable"
finally:
del export
region.mmap_obj.close()
_cleanup_file(region.mmap_path)
def test_backing_file_unlinked_after_barrier(iid):
"""The file must exist until the barrier releases (late joiners need the
name) and be gone right after, so no exit path can leak it."""
+57 -1
View File
@@ -1,9 +1,11 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import mmap
from collections.abc import Collection, Iterable, Mapping
from types import SimpleNamespace
from typing import Any
from unittest.mock import Mock
import msgspec
import numpy as np
@@ -18,6 +20,7 @@ from kvcr.types import (
BlockKey,
CacheTier,
InventoryEvent,
KVCRStartupError,
MemoryRef,
OpEntryResult,
OpEntryStatus,
@@ -172,6 +175,8 @@ def _make_tier(
monkeypatch,
kvcr: RecordingKVCR,
*,
startup_error: BaseException | None = None,
primary_kv_view: memoryview | None = None,
enable_telemetry: bool = False,
secondary_g2_slots: int = 0,
kvcr_service_socket_path: str | None = None,
@@ -195,6 +200,8 @@ def _make_tier(
kvcr.constructor_bindings = bindings
kvcr.framework_control = bindings.framework_control
kvcr.inventory_sink = bindings.inventory_sink
if startup_error is not None:
raise startup_error
return kvcr
monkeypatch.setattr(kvcr_manager, "KVCR", make_kvcr)
@@ -229,12 +236,61 @@ def _make_tier(
self_describing_kv_events=self_describing_kv_events,
),
),
primary_kv_view=memoryview(np.zeros((4, 16), dtype=np.int8)),
primary_kv_view=(
primary_kv_view
if primary_kv_view is not None
else memoryview(np.zeros((4, 16), dtype=np.int8))
),
)
assert isinstance(tier, KVCRSecondaryTierManager)
return tier
@pytest.mark.parametrize(
"error_type", [RuntimeError, KVCRStartupError, KeyboardInterrupt]
)
@pytest.mark.parametrize("secondary_g2_slots", [0, 1])
def test_startup_failure_cleanup(monkeypatch, error_type, secondary_g2_slots):
"""Unsafe failures must hold a buffer export so nothing can unmap memory
native work may still write into. Ordinary failures release everything."""
retained: list[object] = []
monkeypatch.setattr(kvcr_manager, "_RETAINED_NATIVE_BUFFERS", retained)
close = Mock()
monkeypatch.setattr(_StubControlChannel, "close", close)
unsafe = error_type is not RuntimeError
mapping = mmap.mmap(-1, mmap.PAGESIZE)
primary_kv_view = memoryview(mapping)
local_mapping = mmap.mmap(-1, mmap.PAGESIZE) if secondary_g2_slots else None
monkeypatch.setattr(
kvcr_manager, "mmap", SimpleNamespace(mmap=lambda *_: local_mapping)
)
with pytest.raises(error_type, match="startup failed"):
_make_tier(
monkeypatch,
RecordingKVCR(),
startup_error=error_type("startup failed"),
primary_kv_view=primary_kv_view,
secondary_g2_slots=secondary_g2_slots,
)
assert close.call_count == (0 if unsafe else 1)
# Drop the test's own export, leaving only whatever the manager retained:
# the mapping is unmappable afterwards only if that retention covers it.
primary_kv_view.release()
mappings = [mapping] if local_mapping is None else [mapping, local_mapping]
if local_mapping is not None:
assert local_mapping.closed is not unsafe
for region in mappings:
if unsafe:
with pytest.raises(BufferError):
region.close()
retained.clear()
for region in mappings:
region.close()
def test_kvcr_tier_configures_service_for_local_dp_rank(monkeypatch):
"""Keep the control endpoint and guard pool aligned to the local DP rank."""
kvcr = RecordingKVCR()
@@ -368,11 +368,22 @@ class SharedOffloadRegion:
self._views.clear()
self._base = None
if self.mmap_obj:
retained = False
try:
self.mmap_obj.close()
except BufferError:
# A native user still exports this mapping, so unmapping would
# let the NIC write into reused address space. Keep it mapped;
# dropping the fd and the pathname below is still safe.
retained = True
logger.warning(
"Keeping %s mapped: buffers are still exported to native users",
self.mmap_path,
)
except Exception:
logger.warning("Failed to close mmap_obj", exc_info=True)
self.mmap_obj = None
if not retained:
self.mmap_obj = None
if self.fd is not None:
try:
os.close(self.fd)
+18 -1
View File
@@ -42,6 +42,7 @@ from kvcr.types import (
BlockKey,
CacheTier,
InventoryEvent,
KVCRStartupError,
MemoryRef,
OpHandle,
PinRequestId,
@@ -262,6 +263,12 @@ class _VllmKeyAdapter:
_JobState = tuple[int, int, bool, set[OffloadKey] | None]
# Held for the process lifetime after a startup failure that could not stop
# native work: an export blocks close() and keeps the mapping alive against GC,
# so nothing can unmap memory NIXL may still be writing into.
_RETAINED_NATIVE_BUFFERS: list[object] = []
class KVCRSecondaryTierManager(SecondaryTierManager):
"""Secondary tier wrapper around the KVCR KV P2P API."""
@@ -447,7 +454,17 @@ class KVCRSecondaryTierManager(SecondaryTierManager):
),
guard_config,
)
except BaseException:
except BaseException as exc:
if not isinstance(exc, Exception) or isinstance(exc, KVCRStartupError):
# Native work may still touch these.
_RETAINED_NATIVE_BUFFERS.append(
ctypes.c_char.from_buffer(primary_kv_view)
)
if local_mapping is not None:
_RETAINED_NATIVE_BUFFERS.append(
ctypes.c_char.from_buffer(local_mapping)
)
raise
control.close()
if local_mapping is not None:
local_mapping.close()