[KV Offload] Pin only each worker's own offload slots (#59544)
Signed-off-by: Itay Etelis <92247226+Etelis@users.noreply.github.com> Co-authored-by: Robert Shaw <114415538+robertgshaw2-redhat@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Robert Shaw
Codex
Kevin H. Luu
Or Ozeri
parent
6605802904
commit
5341b5ba7d
@@ -10,12 +10,14 @@ import os
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, call
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.v1.kv_offload.cpu import gpu_worker
|
||||
from vllm.v1.kv_offload.cpu import host_register as host_register_module
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as region_module
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import (
|
||||
SharedOffloadRegion,
|
||||
@@ -827,18 +829,23 @@ def host_register(monkeypatch):
|
||||
register = MagicMock(return_value=True)
|
||||
unregister = MagicMock()
|
||||
monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE)
|
||||
monkeypatch.setattr(gpu_worker, "MIN_HOST_REGISTER_SAVED_BYTES", PAGE_SIZE)
|
||||
monkeypatch.setattr(gpu_worker, "host_register", register)
|
||||
monkeypatch.setattr(gpu_worker, "host_unregister", unregister)
|
||||
monkeypatch.setattr(region_module, "host_unregister", unregister)
|
||||
return register, unregister
|
||||
|
||||
|
||||
def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register):
|
||||
@pytest.mark.parametrize("worker_slot_only", [False, True])
|
||||
def test_pin_mmap_region_registers_row_aligned_chunks(
|
||||
iid, host_register, worker_slot_only
|
||||
):
|
||||
"""Chunks end on row boundaries: 3-page rows under a 7-page cap register
|
||||
as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1."""
|
||||
as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1. A slot
|
||||
spanning the whole row registers the same way."""
|
||||
register, unregister = host_register
|
||||
with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region:
|
||||
gpu_worker.pin_mmap_region(region)
|
||||
gpu_worker.pin_mmap_region(region, worker_slot_only=worker_slot_only)
|
||||
base = region._base.data_ptr()
|
||||
assert register.call_args_list == [
|
||||
call(base, 6 * PAGE_SIZE),
|
||||
@@ -854,6 +861,99 @@ def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register):
|
||||
]
|
||||
|
||||
|
||||
def test_pin_mmap_region_worker_slot_only_registers_own_slots(
|
||||
iid, host_register, monkeypatch
|
||||
):
|
||||
"""Rank 1's 1.5-page slot in 4-page rows registers pages [1, 3) of each row."""
|
||||
register, _ = host_register
|
||||
populate = MagicMock()
|
||||
monkeypatch.setattr(region_module, "_get_populate_write_fn", lambda _: populate)
|
||||
region = SharedOffloadRegion(
|
||||
engine_id=iid,
|
||||
num_chunks=3,
|
||||
rank=1,
|
||||
kv_bytes_per_chunk=4 * PAGE_SIZE,
|
||||
cpu_page_size=PAGE_SIZE + PAGE_SIZE // 2,
|
||||
)
|
||||
try:
|
||||
gpu_worker.pin_mmap_region(region, worker_slot_only=True)
|
||||
base = region._base.data_ptr() + PAGE_SIZE
|
||||
assert register.call_args_list == [
|
||||
call(base + row * 4 * PAGE_SIZE, 2 * PAGE_SIZE) for row in range(3)
|
||||
]
|
||||
assert populate.call_args_list == [
|
||||
call(region.mmap_obj, (1 + row * 4) * PAGE_SIZE, 2 * PAGE_SIZE)
|
||||
for row in range(3)
|
||||
]
|
||||
finally:
|
||||
region.cleanup()
|
||||
_cleanup_file(region.mmap_path)
|
||||
|
||||
|
||||
def _unmapped_registration_region(num_chunks, slot_size, num_workers):
|
||||
"""Describe a large region without allocating it; registration is mocked."""
|
||||
region = object.__new__(SharedOffloadRegion)
|
||||
region.rank = 1
|
||||
region.num_chunks = num_chunks
|
||||
region.page_size = PAGE_SIZE
|
||||
region.cpu_page_size = slot_size
|
||||
region._row_stride = num_workers * slot_size
|
||||
region.total_size_bytes = region.num_chunks * region._row_stride
|
||||
region._base = SimpleNamespace(data_ptr=lambda: 0x100000)
|
||||
region.pinned_addresses = []
|
||||
region.is_pinned = False
|
||||
return region
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"num_rows,slot_mib,own_slots",
|
||||
[(2, 2, False), (2, 3, True), ((1 << 16) + 1, 3, True)],
|
||||
)
|
||||
def test_pin_mmap_region_selects_by_savings_not_row_count(
|
||||
monkeypatch, num_rows, slot_mib, own_slots
|
||||
):
|
||||
"""Small savings favor whole-region calls; large row counts do not disable slots."""
|
||||
slot = slot_mib * 1024**2
|
||||
region = _unmapped_registration_region(num_rows, slot, 2)
|
||||
registrations = []
|
||||
|
||||
def register(ptr, size):
|
||||
registrations.append((ptr, size))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(gpu_worker, "host_register", register)
|
||||
gpu_worker.pin_mmap_region(region, worker_slot_only=True)
|
||||
|
||||
assert region.is_pinned
|
||||
assert registrations == (
|
||||
[(0x100000 + (2 * row + 1) * slot, slot) for row in range(num_rows)]
|
||||
if own_slots
|
||||
else [(0x100000, region.total_size_bytes)]
|
||||
)
|
||||
|
||||
|
||||
def test_host_registration_reuses_cudart(monkeypatch):
|
||||
"""Registering and releasing multiple slots constructs one runtime wrapper."""
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostRegister.return_value = 0
|
||||
cudart.cudaHostUnregister.return_value = 0
|
||||
constructor = MagicMock(return_value=cudart)
|
||||
monkeypatch.setattr(host_register_module, "CudaRTLibrary", constructor)
|
||||
monkeypatch.setattr(
|
||||
host_register_module.current_platform, "is_cuda_alike", lambda: True
|
||||
)
|
||||
host_register_module._get_cudart.cache_clear()
|
||||
try:
|
||||
for ptr in (0x100000, 0x200000):
|
||||
assert host_register_module.host_register(ptr, PAGE_SIZE)
|
||||
host_register_module.host_unregister(ptr)
|
||||
constructor.assert_called_once_with()
|
||||
assert cudart.cudaHostRegister.call_count == 2
|
||||
assert cudart.cudaHostUnregister.call_count == 2
|
||||
finally:
|
||||
host_register_module._get_cudart.cache_clear()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_at", [0, 1, 2])
|
||||
def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at):
|
||||
"""A failed chunk unregisters the chunks before it on the same range, so
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
import functools
|
||||
import time
|
||||
from collections import deque
|
||||
from collections.abc import Sequence
|
||||
from collections.abc import Iterator, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import NamedTuple
|
||||
|
||||
@@ -196,10 +196,14 @@ def _canonical_block_sizes(
|
||||
|
||||
# Bound registration size to avoid driver limits on large host allocations.
|
||||
MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3
|
||||
# Amortize per-call overhead with the bytes not registered in each row.
|
||||
MIN_HOST_REGISTER_SAVED_BYTES = 3 * 1024**2
|
||||
|
||||
|
||||
def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
"""Register row-aligned chunks, rolling back on failure."""
|
||||
def pin_mmap_region(
|
||||
region: SharedOffloadRegion, worker_slot_only: bool = False
|
||||
) -> None:
|
||||
"""Register row-aligned chunks or this worker's slots, rolling back on failure."""
|
||||
rank = region.rank
|
||||
base_ptr = region._base.data_ptr()
|
||||
total_size = region.total_size_bytes
|
||||
@@ -208,15 +212,40 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
# registrations.
|
||||
rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1)
|
||||
chunk_size = rows_per_chunk * region._row_stride
|
||||
ranges: Iterator[tuple[int, int]] = (
|
||||
(offset, min(chunk_size, total_size - offset))
|
||||
for offset in range(0, total_size, chunk_size)
|
||||
)
|
||||
stride, slot, page = region._row_stride, region.cpu_page_size, region.page_size
|
||||
min_savings = max(page, MIN_HOST_REGISTER_SAVED_BYTES)
|
||||
if worker_slot_only and region.rank is not None and stride - slot >= min_savings:
|
||||
# Round each slot out to pages; the worker's next slot is a page away.
|
||||
ranges = region.iter_worker_slot_ranges()
|
||||
else:
|
||||
reason = (
|
||||
"whole-region layout requested"
|
||||
if not worker_slot_only
|
||||
else "no worker rank"
|
||||
if region.rank is None
|
||||
else f"less than {min_savings} bytes saved per row"
|
||||
)
|
||||
logger.info(
|
||||
"Registering full CPU offload region rank=%s (row=%d, slot=%d bytes): %s",
|
||||
rank,
|
||||
stride,
|
||||
slot,
|
||||
reason,
|
||||
)
|
||||
|
||||
# Register, drain and roll back through the same helper, so a failed
|
||||
# chunk leaves neither a pending error nor a partly pinned region.
|
||||
addresses: list[int] = []
|
||||
for offset in range(0, total_size, chunk_size):
|
||||
registered_bytes = 0
|
||||
for offset, size in ranges:
|
||||
address = base_ptr + offset
|
||||
size = min(chunk_size, total_size - offset)
|
||||
if host_register(address, size):
|
||||
addresses.append(address)
|
||||
registered_bytes += size
|
||||
continue
|
||||
logger.warning(
|
||||
"host_register failed for rank=%d at %.2f of %.2f GB; "
|
||||
@@ -234,7 +263,7 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
logger.debug(
|
||||
"Host-registered mmap region rank=%d %.2f GB in %d chunk(s)",
|
||||
rank,
|
||||
total_size / 1e9,
|
||||
registered_bytes / 1e9,
|
||||
len(addresses),
|
||||
)
|
||||
|
||||
@@ -785,7 +814,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
pin_memory = PIN_MEMORY
|
||||
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
|
||||
if mmap_region is not None and pin_memory:
|
||||
pin_mmap_region(mmap_region)
|
||||
pin_mmap_region(mmap_region, worker_slot_only=not canonical_layout)
|
||||
host_memory_is_pinned = pin_memory and (
|
||||
mmap_region is None or mmap_region.is_pinned
|
||||
)
|
||||
|
||||
@@ -9,6 +9,8 @@ pure bandwidth optimization: when registration is unavailable the transfers
|
||||
still run correctly, just as pageable (slower) copies.
|
||||
"""
|
||||
|
||||
import functools
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
|
||||
@@ -18,9 +20,14 @@ from vllm.platforms import current_platform
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@functools.cache
|
||||
def _get_cudart() -> CudaRTLibrary:
|
||||
return CudaRTLibrary()
|
||||
|
||||
|
||||
def _load_cudart() -> CudaRTLibrary | None:
|
||||
try:
|
||||
return CudaRTLibrary()
|
||||
return _get_cudart()
|
||||
except (AssertionError, AttributeError, OSError):
|
||||
logger.warning(
|
||||
"Could not load the CUDA runtime for host registration; "
|
||||
|
||||
@@ -5,7 +5,7 @@ import errno
|
||||
import mmap
|
||||
import os
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterator
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -106,6 +106,7 @@ class SharedOffloadRegion:
|
||||
self.num_chunks = num_chunks
|
||||
self._row_stride = kv_bytes_per_chunk
|
||||
self.total_size_bytes = self.num_chunks * self._row_stride
|
||||
self.cpu_page_size = cpu_page_size
|
||||
|
||||
self.mmap_path = f"/dev/shm/vllm_offload_{engine_id}.mmap"
|
||||
self._creator = False # set True only if this worker creates the file
|
||||
@@ -208,15 +209,9 @@ class SharedOffloadRegion:
|
||||
|
||||
if rank is not None:
|
||||
# Populate only this worker's pages (one slot per chunk row).
|
||||
worker_offset = rank * cpu_page_size
|
||||
_t0 = time.perf_counter()
|
||||
page_size = self.page_size
|
||||
for chunk in range(num_chunks):
|
||||
raw_offset = chunk * self._row_stride + worker_offset
|
||||
aligned_offset = (raw_offset // page_size) * page_size
|
||||
end = raw_offset + cpu_page_size
|
||||
aligned_length = end - aligned_offset
|
||||
populate_write_fn(self.mmap_obj, aligned_offset, aligned_length)
|
||||
for offset, length in self.iter_worker_slot_ranges():
|
||||
populate_write_fn(self.mmap_obj, offset, length)
|
||||
logger.debug(
|
||||
"MADV_POPULATE_WRITE loop: %d chunks in %.3f s",
|
||||
num_chunks,
|
||||
@@ -230,6 +225,16 @@ class SharedOffloadRegion:
|
||||
"MADV_POPULATE_WRITE entire region: %.3f s", time.perf_counter() - _t0
|
||||
)
|
||||
|
||||
def iter_worker_slot_ranges(self) -> Iterator[tuple[int, int]]:
|
||||
"""Yield this worker's slots as page-aligned (offset, length) ranges."""
|
||||
assert self.rank is not None
|
||||
worker_offset = self.rank * self.cpu_page_size
|
||||
for row in range(self.num_chunks):
|
||||
start = row * self._row_stride + worker_offset
|
||||
aligned = start - start % self.page_size
|
||||
end = round_up(start + self.cpu_page_size, self.page_size)
|
||||
yield aligned, end - aligned
|
||||
|
||||
@property
|
||||
def base_tensor(self) -> torch.Tensor:
|
||||
if self._base is None:
|
||||
|
||||
Reference in New Issue
Block a user