[KV Offload] Pin only each worker's own offload slots (#59544)

Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com>
Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
Itay Etelis
2026-10-15 08:49:42 +03:00
committed by GitHub
co-authored by Robert Shaw Codex Kevin H. Luu Or Ozeri
parent 6605802904
commit 5341b5ba7d
4 changed files with 161 additions and 20 deletions
@@ -10,12 +10,14 @@ import os
import threading
import time
import uuid
from types import SimpleNamespace
from unittest.mock import MagicMock, call
import pytest
from vllm.utils.system_utils import get_mp_context
from vllm.v1.kv_offload.cpu import gpu_worker
from vllm.v1.kv_offload.cpu import host_register as host_register_module
from vllm.v1.kv_offload.cpu import shared_offload_region as region_module
from vllm.v1.kv_offload.cpu.shared_offload_region import (
SharedOffloadRegion,
@@ -827,18 +829,23 @@ def host_register(monkeypatch):
register = MagicMock(return_value=True)
unregister = MagicMock()
monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE)
monkeypatch.setattr(gpu_worker, "MIN_HOST_REGISTER_SAVED_BYTES", PAGE_SIZE)
monkeypatch.setattr(gpu_worker, "host_register", register)
monkeypatch.setattr(gpu_worker, "host_unregister", unregister)
monkeypatch.setattr(region_module, "host_unregister", unregister)
return register, unregister
def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register):
@pytest.mark.parametrize("worker_slot_only", [False, True])
def test_pin_mmap_region_registers_row_aligned_chunks(
iid, host_register, worker_slot_only
):
"""Chunks end on row boundaries: 3-page rows under a 7-page cap register
as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1."""
as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1. A slot
spanning the whole row registers the same way."""
register, unregister = host_register
with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region:
gpu_worker.pin_mmap_region(region)
gpu_worker.pin_mmap_region(region, worker_slot_only=worker_slot_only)
base = region._base.data_ptr()
assert register.call_args_list == [
call(base, 6 * PAGE_SIZE),
@@ -854,6 +861,99 @@ def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register):
]
def test_pin_mmap_region_worker_slot_only_registers_own_slots(
iid, host_register, monkeypatch
):
"""Rank 1's 1.5-page slot in 4-page rows registers pages [1, 3) of each row."""
register, _ = host_register
populate = MagicMock()
monkeypatch.setattr(region_module, "_get_populate_write_fn", lambda _: populate)
region = SharedOffloadRegion(
engine_id=iid,
num_chunks=3,
rank=1,
kv_bytes_per_chunk=4 * PAGE_SIZE,
cpu_page_size=PAGE_SIZE + PAGE_SIZE // 2,
)
try:
gpu_worker.pin_mmap_region(region, worker_slot_only=True)
base = region._base.data_ptr() + PAGE_SIZE
assert register.call_args_list == [
call(base + row * 4 * PAGE_SIZE, 2 * PAGE_SIZE) for row in range(3)
]
assert populate.call_args_list == [
call(region.mmap_obj, (1 + row * 4) * PAGE_SIZE, 2 * PAGE_SIZE)
for row in range(3)
]
finally:
region.cleanup()
_cleanup_file(region.mmap_path)
def _unmapped_registration_region(num_chunks, slot_size, num_workers):
"""Describe a large region without allocating it; registration is mocked."""
region = object.__new__(SharedOffloadRegion)
region.rank = 1
region.num_chunks = num_chunks
region.page_size = PAGE_SIZE
region.cpu_page_size = slot_size
region._row_stride = num_workers * slot_size
region.total_size_bytes = region.num_chunks * region._row_stride
region._base = SimpleNamespace(data_ptr=lambda: 0x100000)
region.pinned_addresses = []
region.is_pinned = False
return region
@pytest.mark.parametrize(
"num_rows,slot_mib,own_slots",
[(2, 2, False), (2, 3, True), ((1 << 16) + 1, 3, True)],
)
def test_pin_mmap_region_selects_by_savings_not_row_count(
monkeypatch, num_rows, slot_mib, own_slots
):
"""Small savings favor whole-region calls; large row counts do not disable slots."""
slot = slot_mib * 1024**2
region = _unmapped_registration_region(num_rows, slot, 2)
registrations = []
def register(ptr, size):
registrations.append((ptr, size))
return True
monkeypatch.setattr(gpu_worker, "host_register", register)
gpu_worker.pin_mmap_region(region, worker_slot_only=True)
assert region.is_pinned
assert registrations == (
[(0x100000 + (2 * row + 1) * slot, slot) for row in range(num_rows)]
if own_slots
else [(0x100000, region.total_size_bytes)]
)
def test_host_registration_reuses_cudart(monkeypatch):
"""Registering and releasing multiple slots constructs one runtime wrapper."""
cudart = MagicMock()
cudart.cudaHostRegister.return_value = 0
cudart.cudaHostUnregister.return_value = 0
constructor = MagicMock(return_value=cudart)
monkeypatch.setattr(host_register_module, "CudaRTLibrary", constructor)
monkeypatch.setattr(
host_register_module.current_platform, "is_cuda_alike", lambda: True
)
host_register_module._get_cudart.cache_clear()
try:
for ptr in (0x100000, 0x200000):
assert host_register_module.host_register(ptr, PAGE_SIZE)
host_register_module.host_unregister(ptr)
constructor.assert_called_once_with()
assert cudart.cudaHostRegister.call_count == 2
assert cudart.cudaHostUnregister.call_count == 2
finally:
host_register_module._get_cudart.cache_clear()
@pytest.mark.parametrize("fail_at", [0, 1, 2])
def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at):
"""A failed chunk unregisters the chunks before it on the same range, so
+36 -7
View File
@@ -3,7 +3,7 @@
import functools
import time
from collections import deque
from collections.abc import Sequence
from collections.abc import Iterator, Sequence
from dataclasses import dataclass
from typing import NamedTuple
@@ -196,10 +196,14 @@ def _canonical_block_sizes(
# Bound registration size to avoid driver limits on large host allocations.
MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3
# Amortize per-call overhead with the bytes not registered in each row.
MIN_HOST_REGISTER_SAVED_BYTES = 3 * 1024**2
def pin_mmap_region(region: SharedOffloadRegion) -> None:
"""Register row-aligned chunks, rolling back on failure."""
def pin_mmap_region(
region: SharedOffloadRegion, worker_slot_only: bool = False
) -> None:
"""Register row-aligned chunks or this worker's slots, rolling back on failure."""
rank = region.rank
base_ptr = region._base.data_ptr()
total_size = region.total_size_bytes
@@ -208,15 +212,40 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
# registrations.
rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1)
chunk_size = rows_per_chunk * region._row_stride
ranges: Iterator[tuple[int, int]] = (
(offset, min(chunk_size, total_size - offset))
for offset in range(0, total_size, chunk_size)
)
stride, slot, page = region._row_stride, region.cpu_page_size, region.page_size
min_savings = max(page, MIN_HOST_REGISTER_SAVED_BYTES)
if worker_slot_only and region.rank is not None and stride - slot >= min_savings:
# Round each slot out to pages; the worker's next slot is a page away.
ranges = region.iter_worker_slot_ranges()
else:
reason = (
"whole-region layout requested"
if not worker_slot_only
else "no worker rank"
if region.rank is None
else f"less than {min_savings} bytes saved per row"
)
logger.info(
"Registering full CPU offload region rank=%s (row=%d, slot=%d bytes): %s",
rank,
stride,
slot,
reason,
)
# Register, drain and roll back through the same helper, so a failed
# chunk leaves neither a pending error nor a partly pinned region.
addresses: list[int] = []
for offset in range(0, total_size, chunk_size):
registered_bytes = 0
for offset, size in ranges:
address = base_ptr + offset
size = min(chunk_size, total_size - offset)
if host_register(address, size):
addresses.append(address)
registered_bytes += size
continue
logger.warning(
"host_register failed for rank=%d at %.2f of %.2f GB; "
@@ -234,7 +263,7 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
logger.debug(
"Host-registered mmap region rank=%d %.2f GB in %d chunk(s)",
rank,
total_size / 1e9,
registered_bytes / 1e9,
len(addresses),
)
@@ -785,7 +814,7 @@ class CPUOffloadingWorker(OffloadingWorker):
pin_memory = PIN_MEMORY
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
if mmap_region is not None and pin_memory:
pin_mmap_region(mmap_region)
pin_mmap_region(mmap_region, worker_slot_only=not canonical_layout)
host_memory_is_pinned = pin_memory and (
mmap_region is None or mmap_region.is_pinned
)
+8 -1
View File
@@ -9,6 +9,8 @@ pure bandwidth optimization: when registration is unavailable the transfers
still run correctly, just as pageable (slower) copies.
"""
import functools
import torch
from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
@@ -18,9 +20,14 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
@functools.cache
def _get_cudart() -> CudaRTLibrary:
return CudaRTLibrary()
def _load_cudart() -> CudaRTLibrary | None:
try:
return CudaRTLibrary()
return _get_cudart()
except (AssertionError, AttributeError, OSError):
logger.warning(
"Could not load the CUDA runtime for host registration; "
@@ -5,7 +5,7 @@ import errno
import mmap
import os
import time
from collections.abc import Callable
from collections.abc import Callable, Iterator
import numpy as np
import torch
@@ -106,6 +106,7 @@ class SharedOffloadRegion:
self.num_chunks = num_chunks
self._row_stride = kv_bytes_per_chunk
self.total_size_bytes = self.num_chunks * self._row_stride
self.cpu_page_size = cpu_page_size
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
self._creator = False # set True only if this worker creates the file
@@ -208,15 +209,9 @@ class SharedOffloadRegion:
if rank is not None:
# Populate only this worker's pages (one slot per chunk row).
worker_offset = rank * cpu_page_size
_t0 = time.perf_counter()
page_size = self.page_size
for chunk in range(num_chunks):
raw_offset = chunk * self._row_stride + worker_offset
aligned_offset = (raw_offset // page_size) * page_size
end = raw_offset + cpu_page_size
aligned_length = end - aligned_offset
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
for offset, length in self.iter_worker_slot_ranges():
populate_write_fn(self.mmap_obj, offset, length)
logger.debug(
"MADV_POPULATE_WRITE loop: %d chunks in %.3f s",
num_chunks,
@@ -230,6 +225,16 @@ class SharedOffloadRegion:
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
)
def iter_worker_slot_ranges(self) -> Iterator[tuple[int, int]]:
"""Yield this worker's slots as page-aligned (offset, length) ranges."""
assert self.rank is not None
worker_offset = self.rank * self.cpu_page_size
for row in range(self.num_chunks):
start = row * self._row_stride + worker_offset
aligned = start - start % self.page_size
end = round_up(start + self.cpu_page_size, self.page_size)
yield aligned, end - aligned
@property
def base_tensor(self) -> torch.Tensor:
if self._base is None: