[Bugfix][KV Offload] Register the offload region in chunks (#51081)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Colton Ottley <coltonottley@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude
Colton Ottley
Or Ozeri
parent
08633cb5cd
commit
40b40d1d39
@@ -241,6 +241,7 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int):
|
||||
# The Triton load path dereferences CPU pointers on the GPU, which is
|
||||
# only legal on pinned memory; production pins via CPUOffloadingWorker
|
||||
pin_mmap_region(region)
|
||||
assert region.is_pinned
|
||||
return region.create_next_canonical_view(_CANONICAL_PAGE)
|
||||
|
||||
try:
|
||||
|
||||
@@ -49,6 +49,22 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not gpu_worker.HAS_TRITON, reason="Requires Triton")
|
||||
def test_unpinned_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
"""The Triton load path dereferences CPU pointers on the GPU, so pageable
|
||||
host memory takes the DMA path even for pages where Triton would win."""
|
||||
monkeypatch.setattr(gpu_worker.current_platform, "is_xpu", lambda: False)
|
||||
monkeypatch.setattr(gpu_worker.current_platform, "is_rocm", lambda: False)
|
||||
|
||||
refs = [[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=512)]]
|
||||
assert gpu_worker._select_swap_blocks_fn(refs, gpu_to_cpu=False) is not (
|
||||
ops.swap_blocks_batch
|
||||
)
|
||||
assert gpu_worker._select_swap_blocks_fn(
|
||||
refs, gpu_to_cpu=False, host_memory_is_pinned=False
|
||||
) is (ops.swap_blocks_batch)
|
||||
|
||||
|
||||
def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None:
|
||||
"""Both directions drain before the worker releases its shared region."""
|
||||
worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker)
|
||||
|
||||
@@ -15,6 +15,7 @@ from unittest.mock import MagicMock, call
|
||||
import pytest
|
||||
|
||||
from vllm.utils.system_utils import get_mp_context
|
||||
from vllm.v1.kv_offload.cpu import gpu_worker
|
||||
from vllm.v1.kv_offload.cpu import shared_offload_region as region_module
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import (
|
||||
SharedOffloadRegion,
|
||||
@@ -788,6 +789,87 @@ def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch):
|
||||
]
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def host_register(monkeypatch):
|
||||
"""Pretend to run on CUDA with host registrations capped at seven pages.
|
||||
|
||||
Registration goes to a mock CudaRTLibrary and the region's cleanup to a
|
||||
separate mock torch runtime. Both stay installed through the cleanup that
|
||||
``_region`` runs on exit.
|
||||
"""
|
||||
cudart = MagicMock()
|
||||
cudart.cudaHostRegister.return_value = 0
|
||||
cudart.cudaHostUnregister.return_value = 0
|
||||
cudart.cudaGetLastError.return_value = 2
|
||||
torch_cudart = MagicMock()
|
||||
torch_cudart.cudaHostUnregister.return_value = MagicMock(value=0)
|
||||
monkeypatch.setattr(gpu_worker.current_platform, "is_cuda_alike", lambda: True)
|
||||
monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE)
|
||||
monkeypatch.setattr(gpu_worker, "CudaRTLibrary", lambda: cudart)
|
||||
monkeypatch.setattr(region_module.torch.cuda, "cudart", lambda: torch_cudart)
|
||||
return cudart, torch_cudart
|
||||
|
||||
|
||||
def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register):
|
||||
"""Chunks end on row boundaries: 3-page rows under a 7-page cap register
|
||||
as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1."""
|
||||
cudart, torch_cudart = host_register
|
||||
with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region:
|
||||
gpu_worker.pin_mmap_region(region)
|
||||
base = region._base.data_ptr()
|
||||
assert cudart.mock_calls == [
|
||||
call.cudaHostRegister(base, 6 * PAGE_SIZE),
|
||||
call.cudaHostRegister(base + 6 * PAGE_SIZE, 6 * PAGE_SIZE),
|
||||
call.cudaHostRegister(base + 12 * PAGE_SIZE, 3 * PAGE_SIZE),
|
||||
]
|
||||
assert region.is_pinned
|
||||
|
||||
assert torch_cudart.cudaHostUnregister.call_args_list == [
|
||||
call(base + 12 * PAGE_SIZE),
|
||||
call(base + 6 * PAGE_SIZE),
|
||||
call(base),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("fail_at", [0, 1, 2])
|
||||
def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at):
|
||||
"""A failed chunk drains its pending error and unregisters the chunks
|
||||
before it on the same handle, so the region is pinned whole or not at all."""
|
||||
cudart, torch_cudart = host_register
|
||||
cudart.cudaHostRegister.side_effect = [0] * fail_at + [2]
|
||||
with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region:
|
||||
gpu_worker.pin_mmap_region(region)
|
||||
base = region._base.data_ptr()
|
||||
assert not region.is_pinned
|
||||
assert region.pinned_addresses == []
|
||||
|
||||
registered = [base + i * 6 * PAGE_SIZE for i in range(fail_at)]
|
||||
failed_size = min(6, 15 - 6 * fail_at) * PAGE_SIZE
|
||||
assert cudart.mock_calls == [
|
||||
*(call.cudaHostRegister(address, 6 * PAGE_SIZE) for address in registered),
|
||||
call.cudaHostRegister(base + fail_at * 6 * PAGE_SIZE, failed_size),
|
||||
call.cudaGetLastError(),
|
||||
*(call.cudaHostUnregister(address) for address in reversed(registered)),
|
||||
]
|
||||
torch_cudart.cudaHostUnregister.assert_not_called()
|
||||
|
||||
|
||||
def test_pin_mmap_region_without_cuda_runtime_stays_pageable(
|
||||
iid, host_register, monkeypatch
|
||||
):
|
||||
"""A CUDA runtime that cannot be loaded leaves the region pageable instead
|
||||
of failing worker startup."""
|
||||
|
||||
def missing_runtime():
|
||||
raise AssertionError("libcudart is not loaded in the current process")
|
||||
|
||||
monkeypatch.setattr(gpu_worker, "CudaRTLibrary", missing_runtime)
|
||||
with _region(iid) as region:
|
||||
gpu_worker.pin_mmap_region(region)
|
||||
assert not region.is_pinned
|
||||
assert region.pinned_addresses == []
|
||||
|
||||
|
||||
def test_cleanup_after_create_next_worker_view_releases_mmap(iid):
|
||||
"""cleanup() must close the mmap even after create_next_worker_view was called.
|
||||
create_next_worker_view returns a view that shares storage with _base; both must be
|
||||
|
||||
@@ -78,6 +78,16 @@ class CudaRTLibrary:
|
||||
cudaError_t,
|
||||
[ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint],
|
||||
),
|
||||
# cudaError_t cudaHostRegister ( void* ptr, size_t size, unsigned int flags )
|
||||
Function(
|
||||
"cudaHostRegister",
|
||||
cudaError_t,
|
||||
[ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint],
|
||||
),
|
||||
# cudaError_t cudaHostUnregister ( void* ptr )
|
||||
Function("cudaHostUnregister", cudaError_t, [ctypes.c_void_p]),
|
||||
# cudaError_t cudaGetLastError ( void )
|
||||
Function("cudaGetLastError", cudaError_t, []),
|
||||
]
|
||||
|
||||
# https://rocm.docs.amd.com/projects/HIPIFY/en/latest/tables/CUDA_Runtime_API_functions_supported_by_HIP.html # noqa
|
||||
@@ -92,6 +102,9 @@ class CudaRTLibrary:
|
||||
"cudaMemcpy": "hipMemcpy",
|
||||
"cudaIpcGetMemHandle": "hipIpcGetMemHandle",
|
||||
"cudaIpcOpenMemHandle": "hipIpcOpenMemHandle",
|
||||
"cudaHostRegister": "hipHostRegister",
|
||||
"cudaHostUnregister": "hipHostUnregister",
|
||||
"cudaGetLastError": "hipGetLastError",
|
||||
}
|
||||
|
||||
# class attribute to store the mapping from the path to the library
|
||||
@@ -185,3 +198,16 @@ class CudaRTLibrary:
|
||||
)
|
||||
)
|
||||
return devPtr
|
||||
|
||||
def cudaHostRegister(self, ptr: int, size: int, flags: int = 0) -> int:
|
||||
"""Return the raw error code instead of raising, since callers can
|
||||
fall back to pageable memory. Drain a failure with cudaGetLastError."""
|
||||
return self.funcs["cudaHostRegister"](ctypes.c_void_p(ptr), size, flags)
|
||||
|
||||
def cudaHostUnregister(self, ptr: int) -> int:
|
||||
"""Return the raw error code instead of raising."""
|
||||
return self.funcs["cudaHostUnregister"](ctypes.c_void_p(ptr))
|
||||
|
||||
def cudaGetLastError(self) -> int:
|
||||
"""Return and clear the error pending on this thread."""
|
||||
return self.funcs["cudaGetLastError"]()
|
||||
|
||||
@@ -11,6 +11,7 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, triton
|
||||
@@ -38,10 +39,13 @@ logger = init_logger(__name__)
|
||||
def _select_swap_blocks_fn(
|
||||
layer_refs_per_group: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
host_memory_is_pinned: bool = True,
|
||||
):
|
||||
"""Resolve the swap_blocks function for a handler at init time."""
|
||||
# GPU->CPU is bandwidth-bound; the dedicated copy engine beats Triton.
|
||||
if gpu_to_cpu:
|
||||
# The Triton kernel dereferences CPU pointers on the GPU, which is only
|
||||
# valid for pinned host memory.
|
||||
if gpu_to_cpu or not host_memory_is_pinned:
|
||||
return ops.swap_blocks_batch
|
||||
# Fall back to the C++ DMA path on platforms where Triton isn't usable
|
||||
# (e.g. ROCm host mappings) or where GPU kernels cannot directly
|
||||
@@ -190,8 +194,12 @@ def _canonical_block_sizes(
|
||||
return canonical_bytes_per_block
|
||||
|
||||
|
||||
# Bound registration size to avoid driver limits on large host allocations.
|
||||
MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3
|
||||
|
||||
|
||||
def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
"""Register the entire mmap as CUDA pinned memory via cudaHostRegister."""
|
||||
"""Register row-aligned chunks, rolling back on failure."""
|
||||
if not current_platform.is_cuda_alike():
|
||||
logger.info(
|
||||
"Skipping mmap host registration on %s; cudaHostRegister is only "
|
||||
@@ -201,23 +209,65 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
return
|
||||
|
||||
rank = region.rank
|
||||
try:
|
||||
cudart = CudaRTLibrary()
|
||||
except (AssertionError, AttributeError, OSError):
|
||||
logger.warning(
|
||||
"Could not load the CUDA runtime for host registration on rank=%d; "
|
||||
"the offload region stays pageable",
|
||||
rank,
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
|
||||
base_ptr = region._base.data_ptr()
|
||||
result = torch.cuda.cudart().cudaHostRegister(base_ptr, region.total_size_bytes, 0)
|
||||
if result.value != 0:
|
||||
total_size = region.total_size_bytes
|
||||
# Chunks end on block-row boundaries, which are page aligned, so neither the
|
||||
# driver's page rounding nor any single block transfer straddles two
|
||||
# registrations.
|
||||
rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1)
|
||||
chunk_size = rows_per_chunk * region._row_stride
|
||||
|
||||
# Register, drain and roll back through the same runtime handle, so a
|
||||
# failed chunk leaves neither a pending error nor a partly pinned region.
|
||||
addresses: list[int] = []
|
||||
for offset in range(0, total_size, chunk_size):
|
||||
address = base_ptr + offset
|
||||
size = min(chunk_size, total_size - offset)
|
||||
result = cudart.cudaHostRegister(address, size)
|
||||
if result == 0:
|
||||
addresses.append(address)
|
||||
continue
|
||||
cudart.cudaGetLastError()
|
||||
logger.warning(
|
||||
"cudaHostRegister failed for rank=%d (code=%d) — "
|
||||
"transfers will still work but may be slower (unpinned DMA)",
|
||||
"cudaHostRegister failed for rank=%d at %.2f of %.2f GB (code=%d); "
|
||||
"the offload region stays pageable",
|
||||
rank,
|
||||
offset / 1e9,
|
||||
total_size / 1e9,
|
||||
result,
|
||||
)
|
||||
else:
|
||||
logger.debug(
|
||||
"cudaHostRegister rank=%d %.2f GB",
|
||||
rank,
|
||||
region.total_size_bytes / 1e9,
|
||||
)
|
||||
region.is_pinned = True
|
||||
for registered in reversed(addresses):
|
||||
unregister_result = cudart.cudaHostUnregister(registered)
|
||||
if unregister_result != 0:
|
||||
cudart.cudaGetLastError()
|
||||
logger.warning(
|
||||
"cudaHostUnregister failed for rank=%d at %#x (code=%d); "
|
||||
"that chunk stays registered until the process exits",
|
||||
rank,
|
||||
registered,
|
||||
unregister_result,
|
||||
)
|
||||
return
|
||||
|
||||
region.pinned_addresses.extend(addresses)
|
||||
region.is_pinned = True
|
||||
logger.debug(
|
||||
"cudaHostRegister rank=%d %.2f GB in %d chunk(s)",
|
||||
rank,
|
||||
total_size / 1e9,
|
||||
len(addresses),
|
||||
)
|
||||
|
||||
|
||||
def _new_descriptor_buffers(
|
||||
@@ -248,6 +298,7 @@ class SingleDirectionOffloadingHandler:
|
||||
layer_refs_per_group: list[list[CanonicalKVCacheRef]],
|
||||
gpu_to_cpu: bool,
|
||||
canonical_layout: bool = False,
|
||||
host_memory_is_pinned: bool = True,
|
||||
):
|
||||
"""Initialize a SingleDirectionOffloadingHandler.
|
||||
|
||||
@@ -262,6 +313,8 @@ class SingleDirectionOffloadingHandler:
|
||||
gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU.
|
||||
canonical_layout: if True, CPU pages use the canonical layout
|
||||
described by the refs' mappings.
|
||||
host_memory_is_pinned: whether the CPU tensors are pinned, so GPU
|
||||
kernels may dereference them directly.
|
||||
|
||||
"""
|
||||
assert len(gpu_tensors) == len(cpu_tensors)
|
||||
@@ -299,7 +352,7 @@ class SingleDirectionOffloadingHandler:
|
||||
self.gpu_to_cpu: bool = gpu_to_cpu
|
||||
self.layer_refs_per_group = layer_refs_per_group
|
||||
self._swap_blocks_batch = _select_swap_blocks_fn(
|
||||
layer_refs_per_group, gpu_to_cpu
|
||||
layer_refs_per_group, gpu_to_cpu, host_memory_is_pinned
|
||||
)
|
||||
|
||||
# GPU blocks may be smaller
|
||||
@@ -646,7 +699,7 @@ class SingleDirectionOffloadingHandler:
|
||||
last_event = last_transfer.end_event
|
||||
# assure job will start only after the previous one completes
|
||||
stream.wait_event(last_event)
|
||||
# CPU->GPU reads from host pinned memory, which is never written
|
||||
# CPU->GPU reads from host memory, which is never written
|
||||
# by a concurrent GPU stream, so CU_MEMCPY_SRC_ACCESS_ORDER_ANY is
|
||||
# safe and lets the driver pipeline source reads. GPU->CPU reads
|
||||
# from the live GPU KV cache, which the compute stream keeps
|
||||
@@ -764,6 +817,9 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors))
|
||||
if mmap_region is not None and pin_memory:
|
||||
pin_mmap_region(mmap_region)
|
||||
host_memory_is_pinned = pin_memory and (
|
||||
mmap_region is None or mmap_region.is_pinned
|
||||
)
|
||||
|
||||
canonical_bytes_per_block = (
|
||||
_canonical_block_sizes(kv_caches.group_data_refs, len(kv_caches.tensors))
|
||||
@@ -813,6 +869,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
layer_refs_per_group=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=True,
|
||||
canonical_layout=canonical_layout,
|
||||
host_memory_is_pinned=host_memory_is_pinned,
|
||||
)
|
||||
|
||||
self._load_handler = SingleDirectionOffloadingHandler(
|
||||
@@ -822,6 +879,7 @@ class CPUOffloadingWorker(OffloadingWorker):
|
||||
layer_refs_per_group=kv_caches.group_data_refs,
|
||||
gpu_to_cpu=False,
|
||||
canonical_layout=canonical_layout,
|
||||
host_memory_is_pinned=host_memory_is_pinned,
|
||||
)
|
||||
|
||||
def submit_store(
|
||||
|
||||
Reference in New Issue
Block a user