From 5341b5ba7d10260008c36c06c30fed736aad175f Mon Sep 17 00:00:00 2001 From: Itay Etelis <92247226+Etelis@users.noreply.github.com> Date: Thu, 15 Oct 2026 08:49:42 +0300 Subject: [PATCH] [KV Offload] Pin only each worker's own offload slots (#59544) Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Codex Co-authored-by: Kevin H. Luu Co-authored-by: Or Ozeri --- .../cpu/test_shared_offload_region.py | 106 +++++++++++++++++- vllm/v1/kv_offload/cpu/gpu_worker.py | 43 +++++-- vllm/v1/kv_offload/cpu/host_register.py | 9 +- .../kv_offload/cpu/shared_offload_region.py | 23 ++-- 4 files changed, 161 insertions(+), 20 deletions(-) diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index ada31beac2..21ef6d2761 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -10,12 +10,14 @@ import os import threading import time import uuid +from types import SimpleNamespace from unittest.mock import MagicMock, call import pytest from vllm.utils.system_utils import get_mp_context from vllm.v1.kv_offload.cpu import gpu_worker +from vllm.v1.kv_offload.cpu import host_register as host_register_module from vllm.v1.kv_offload.cpu import shared_offload_region as region_module from vllm.v1.kv_offload.cpu.shared_offload_region import ( SharedOffloadRegion, @@ -827,18 +829,23 @@ def host_register(monkeypatch): register = MagicMock(return_value=True) unregister = MagicMock() monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE) + monkeypatch.setattr(gpu_worker, "MIN_HOST_REGISTER_SAVED_BYTES", PAGE_SIZE) monkeypatch.setattr(gpu_worker, "host_register", register) monkeypatch.setattr(gpu_worker, "host_unregister", unregister) monkeypatch.setattr(region_module, "host_unregister", unregister) return register, unregister -def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register): +@pytest.mark.parametrize("worker_slot_only", [False, True]) +def test_pin_mmap_region_registers_row_aligned_chunks( + iid, host_register, worker_slot_only +): """Chunks end on row boundaries: 3-page rows under a 7-page cap register - as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1.""" + as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1. A slot + spanning the whole row registers the same way.""" register, unregister = host_register with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: - gpu_worker.pin_mmap_region(region) + gpu_worker.pin_mmap_region(region, worker_slot_only=worker_slot_only) base = region._base.data_ptr() assert register.call_args_list == [ call(base, 6 * PAGE_SIZE), @@ -854,6 +861,99 @@ def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register): ] +def test_pin_mmap_region_worker_slot_only_registers_own_slots( + iid, host_register, monkeypatch +): + """Rank 1's 1.5-page slot in 4-page rows registers pages [1, 3) of each row.""" + register, _ = host_register + populate = MagicMock() + monkeypatch.setattr(region_module, "_get_populate_write_fn", lambda _: populate) + region = SharedOffloadRegion( + engine_id=iid, + num_chunks=3, + rank=1, + kv_bytes_per_chunk=4 * PAGE_SIZE, + cpu_page_size=PAGE_SIZE + PAGE_SIZE // 2, + ) + try: + gpu_worker.pin_mmap_region(region, worker_slot_only=True) + base = region._base.data_ptr() + PAGE_SIZE + assert register.call_args_list == [ + call(base + row * 4 * PAGE_SIZE, 2 * PAGE_SIZE) for row in range(3) + ] + assert populate.call_args_list == [ + call(region.mmap_obj, (1 + row * 4) * PAGE_SIZE, 2 * PAGE_SIZE) + for row in range(3) + ] + finally: + region.cleanup() + _cleanup_file(region.mmap_path) + + +def _unmapped_registration_region(num_chunks, slot_size, num_workers): + """Describe a large region without allocating it; registration is mocked.""" + region = object.__new__(SharedOffloadRegion) + region.rank = 1 + region.num_chunks = num_chunks + region.page_size = PAGE_SIZE + region.cpu_page_size = slot_size + region._row_stride = num_workers * slot_size + region.total_size_bytes = region.num_chunks * region._row_stride + region._base = SimpleNamespace(data_ptr=lambda: 0x100000) + region.pinned_addresses = [] + region.is_pinned = False + return region + + +@pytest.mark.parametrize( + "num_rows,slot_mib,own_slots", + [(2, 2, False), (2, 3, True), ((1 << 16) + 1, 3, True)], +) +def test_pin_mmap_region_selects_by_savings_not_row_count( + monkeypatch, num_rows, slot_mib, own_slots +): + """Small savings favor whole-region calls; large row counts do not disable slots.""" + slot = slot_mib * 1024**2 + region = _unmapped_registration_region(num_rows, slot, 2) + registrations = [] + + def register(ptr, size): + registrations.append((ptr, size)) + return True + + monkeypatch.setattr(gpu_worker, "host_register", register) + gpu_worker.pin_mmap_region(region, worker_slot_only=True) + + assert region.is_pinned + assert registrations == ( + [(0x100000 + (2 * row + 1) * slot, slot) for row in range(num_rows)] + if own_slots + else [(0x100000, region.total_size_bytes)] + ) + + +def test_host_registration_reuses_cudart(monkeypatch): + """Registering and releasing multiple slots constructs one runtime wrapper.""" + cudart = MagicMock() + cudart.cudaHostRegister.return_value = 0 + cudart.cudaHostUnregister.return_value = 0 + constructor = MagicMock(return_value=cudart) + monkeypatch.setattr(host_register_module, "CudaRTLibrary", constructor) + monkeypatch.setattr( + host_register_module.current_platform, "is_cuda_alike", lambda: True + ) + host_register_module._get_cudart.cache_clear() + try: + for ptr in (0x100000, 0x200000): + assert host_register_module.host_register(ptr, PAGE_SIZE) + host_register_module.host_unregister(ptr) + constructor.assert_called_once_with() + assert cudart.cudaHostRegister.call_count == 2 + assert cudart.cudaHostUnregister.call_count == 2 + finally: + host_register_module._get_cudart.cache_clear() + + @pytest.mark.parametrize("fail_at", [0, 1, 2]) def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at): """A failed chunk unregisters the chunks before it on the same range, so diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index f6053480a7..134d8c08b0 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -3,7 +3,7 @@ import functools import time from collections import deque -from collections.abc import Sequence +from collections.abc import Iterator, Sequence from dataclasses import dataclass from typing import NamedTuple @@ -196,10 +196,14 @@ def _canonical_block_sizes( # Bound registration size to avoid driver limits on large host allocations. MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3 +# Amortize per-call overhead with the bytes not registered in each row. +MIN_HOST_REGISTER_SAVED_BYTES = 3 * 1024**2 -def pin_mmap_region(region: SharedOffloadRegion) -> None: - """Register row-aligned chunks, rolling back on failure.""" +def pin_mmap_region( + region: SharedOffloadRegion, worker_slot_only: bool = False +) -> None: + """Register row-aligned chunks or this worker's slots, rolling back on failure.""" rank = region.rank base_ptr = region._base.data_ptr() total_size = region.total_size_bytes @@ -208,15 +212,40 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None: # registrations. rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1) chunk_size = rows_per_chunk * region._row_stride + ranges: Iterator[tuple[int, int]] = ( + (offset, min(chunk_size, total_size - offset)) + for offset in range(0, total_size, chunk_size) + ) + stride, slot, page = region._row_stride, region.cpu_page_size, region.page_size + min_savings = max(page, MIN_HOST_REGISTER_SAVED_BYTES) + if worker_slot_only and region.rank is not None and stride - slot >= min_savings: + # Round each slot out to pages; the worker's next slot is a page away. + ranges = region.iter_worker_slot_ranges() + else: + reason = ( + "whole-region layout requested" + if not worker_slot_only + else "no worker rank" + if region.rank is None + else f"less than {min_savings} bytes saved per row" + ) + logger.info( + "Registering full CPU offload region rank=%s (row=%d, slot=%d bytes): %s", + rank, + stride, + slot, + reason, + ) # Register, drain and roll back through the same helper, so a failed # chunk leaves neither a pending error nor a partly pinned region. addresses: list[int] = [] - for offset in range(0, total_size, chunk_size): + registered_bytes = 0 + for offset, size in ranges: address = base_ptr + offset - size = min(chunk_size, total_size - offset) if host_register(address, size): addresses.append(address) + registered_bytes += size continue logger.warning( "host_register failed for rank=%d at %.2f of %.2f GB; " @@ -234,7 +263,7 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None: logger.debug( "Host-registered mmap region rank=%d %.2f GB in %d chunk(s)", rank, - total_size / 1e9, + registered_bytes / 1e9, len(addresses), ) @@ -785,7 +814,7 @@ class CPUOffloadingWorker(OffloadingWorker): pin_memory = PIN_MEMORY logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors)) if mmap_region is not None and pin_memory: - pin_mmap_region(mmap_region) + pin_mmap_region(mmap_region, worker_slot_only=not canonical_layout) host_memory_is_pinned = pin_memory and ( mmap_region is None or mmap_region.is_pinned ) diff --git a/vllm/v1/kv_offload/cpu/host_register.py b/vllm/v1/kv_offload/cpu/host_register.py index 3744885740..c2147a75bf 100644 --- a/vllm/v1/kv_offload/cpu/host_register.py +++ b/vllm/v1/kv_offload/cpu/host_register.py @@ -9,6 +9,8 @@ pure bandwidth optimization: when registration is unavailable the transfers still run correctly, just as pageable (slower) copies. """ +import functools + import torch from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary @@ -18,9 +20,14 @@ from vllm.platforms import current_platform logger = init_logger(__name__) +@functools.cache +def _get_cudart() -> CudaRTLibrary: + return CudaRTLibrary() + + def _load_cudart() -> CudaRTLibrary | None: try: - return CudaRTLibrary() + return _get_cudart() except (AssertionError, AttributeError, OSError): logger.warning( "Could not load the CUDA runtime for host registration; " diff --git a/vllm/v1/kv_offload/cpu/shared_offload_region.py b/vllm/v1/kv_offload/cpu/shared_offload_region.py index 6a7e26b4b8..34faf28b94 100644 --- a/vllm/v1/kv_offload/cpu/shared_offload_region.py +++ b/vllm/v1/kv_offload/cpu/shared_offload_region.py @@ -5,7 +5,7 @@ import errno import mmap import os import time -from collections.abc import Callable +from collections.abc import Callable, Iterator import numpy as np import torch @@ -106,6 +106,7 @@ class SharedOffloadRegion: self.num_chunks = num_chunks self._row_stride = kv_bytes_per_chunk self.total_size_bytes = self.num_chunks * self._row_stride + self.cpu_page_size = cpu_page_size self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap" self._creator = False # set True only if this worker creates the file @@ -208,15 +209,9 @@ class SharedOffloadRegion: if rank is not None: # Populate only this worker's pages (one slot per chunk row). - worker_offset = rank * cpu_page_size _t0 = time.perf_counter() - page_size = self.page_size - for chunk in range(num_chunks): - raw_offset = chunk * self._row_stride + worker_offset - aligned_offset = (raw_offset // page_size) * page_size - end = raw_offset + cpu_page_size - aligned_length = end - aligned_offset - populate_write_fn(self.mmap_obj, aligned_offset, aligned_length) + for offset, length in self.iter_worker_slot_ranges(): + populate_write_fn(self.mmap_obj, offset, length) logger.debug( "MADV_POPULATE_WRITE loop: %d chunks in %.3f s", num_chunks, @@ -230,6 +225,16 @@ class SharedOffloadRegion: "MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0 ) + def iter_worker_slot_ranges(self) -> Iterator[tuple[int, int]]: + """Yield this worker's slots as page-aligned (offset, length) ranges.""" + assert self.rank is not None + worker_offset = self.rank * self.cpu_page_size + for row in range(self.num_chunks): + start = row * self._row_stride + worker_offset + aligned = start - start % self.page_size + end = round_up(start + self.cpu_page_size, self.page_size) + yield aligned, end - aligned + @property def base_tensor(self) -> torch.Tensor: if self._base is None: