diff --git a/tests/v1/kv_offload/cpu/test_canonical_layout.py b/tests/v1/kv_offload/cpu/test_canonical_layout.py index 4594e1eb09..749bd6fac5 100644 --- a/tests/v1/kv_offload/cpu/test_canonical_layout.py +++ b/tests/v1/kv_offload/cpu/test_canonical_layout.py @@ -241,6 +241,7 @@ def test_cross_topology_roundtrip(writer_tp: int, reader_tp: int): # The Triton load path dereferences CPU pointers on the GPU, which is # only legal on pinned memory; production pins via CPUOffloadingWorker pin_mmap_region(region) + assert region.is_pinned return region.create_next_canonical_view(_CANONICAL_PAGE) try: diff --git a/tests/v1/kv_offload/cpu/test_gpu_worker.py b/tests/v1/kv_offload/cpu/test_gpu_worker.py index 5306d05571..731d61a7e9 100644 --- a/tests/v1/kv_offload/cpu/test_gpu_worker.py +++ b/tests/v1/kv_offload/cpu/test_gpu_worker.py @@ -49,6 +49,22 @@ def test_rocm_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: ) +@pytest.mark.skipif(not gpu_worker.HAS_TRITON, reason="Requires Triton") +def test_unpinned_cpu_to_gpu_uses_dma(monkeypatch: pytest.MonkeyPatch) -> None: + """The Triton load path dereferences CPU pointers on the GPU, so pageable + host memory takes the DMA path even for pages where Triton would win.""" + monkeypatch.setattr(gpu_worker.current_platform, "is_xpu", lambda: False) + monkeypatch.setattr(gpu_worker.current_platform, "is_rocm", lambda: False) + + refs = [[CanonicalKVCacheRef(tensor_idx=0, page_size_bytes=512)]] + assert gpu_worker._select_swap_blocks_fn(refs, gpu_to_cpu=False) is not ( + ops.swap_blocks_batch + ) + assert gpu_worker._select_swap_blocks_fn( + refs, gpu_to_cpu=False, host_memory_is_pinned=False + ) is (ops.swap_blocks_batch) + + def test_worker_shutdown_releases_region_and_runs_both_handlers() -> None: """Both directions drain before the worker releases its shared region.""" worker = CPUOffloadingWorker.__new__(CPUOffloadingWorker) diff --git a/tests/v1/kv_offload/cpu/test_shared_offload_region.py b/tests/v1/kv_offload/cpu/test_shared_offload_region.py index 4f22d122db..f84e1bd729 100644 --- a/tests/v1/kv_offload/cpu/test_shared_offload_region.py +++ b/tests/v1/kv_offload/cpu/test_shared_offload_region.py @@ -15,6 +15,7 @@ from unittest.mock import MagicMock, call import pytest from vllm.utils.system_utils import get_mp_context +from vllm.v1.kv_offload.cpu import gpu_worker from vllm.v1.kv_offload.cpu import shared_offload_region as region_module from vllm.v1.kv_offload.cpu.shared_offload_region import ( SharedOffloadRegion, @@ -788,6 +789,87 @@ def test_cleanup_unregisters_every_pinned_chunk(iid, monkeypatch): ] +@pytest.fixture +def host_register(monkeypatch): + """Pretend to run on CUDA with host registrations capped at seven pages. + + Registration goes to a mock CudaRTLibrary and the region's cleanup to a + separate mock torch runtime. Both stay installed through the cleanup that + ``_region`` runs on exit. + """ + cudart = MagicMock() + cudart.cudaHostRegister.return_value = 0 + cudart.cudaHostUnregister.return_value = 0 + cudart.cudaGetLastError.return_value = 2 + torch_cudart = MagicMock() + torch_cudart.cudaHostUnregister.return_value = MagicMock(value=0) + monkeypatch.setattr(gpu_worker.current_platform, "is_cuda_alike", lambda: True) + monkeypatch.setattr(gpu_worker, "MAX_HOST_REGISTER_CHUNK_BYTES", 7 * PAGE_SIZE) + monkeypatch.setattr(gpu_worker, "CudaRTLibrary", lambda: cudart) + monkeypatch.setattr(region_module.torch.cuda, "cudart", lambda: torch_cudart) + return cudart, torch_cudart + + +def test_pin_mmap_region_registers_row_aligned_chunks(iid, host_register): + """Chunks end on row boundaries: 3-page rows under a 7-page cap register + as 6 + 6 + 3 pages, where a raw byte cap would give 7 + 7 + 1.""" + cudart, torch_cudart = host_register + with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: + gpu_worker.pin_mmap_region(region) + base = region._base.data_ptr() + assert cudart.mock_calls == [ + call.cudaHostRegister(base, 6 * PAGE_SIZE), + call.cudaHostRegister(base + 6 * PAGE_SIZE, 6 * PAGE_SIZE), + call.cudaHostRegister(base + 12 * PAGE_SIZE, 3 * PAGE_SIZE), + ] + assert region.is_pinned + + assert torch_cudart.cudaHostUnregister.call_args_list == [ + call(base + 12 * PAGE_SIZE), + call(base + 6 * PAGE_SIZE), + call(base), + ] + + +@pytest.mark.parametrize("fail_at", [0, 1, 2]) +def test_pin_mmap_region_failure_leaves_region_pageable(iid, host_register, fail_at): + """A failed chunk drains its pending error and unregisters the chunks + before it on the same handle, so the region is pinned whole or not at all.""" + cudart, torch_cudart = host_register + cudart.cudaHostRegister.side_effect = [0] * fail_at + [2] + with _region(iid, num_chunks=5, cpu_page_size=3 * PAGE_SIZE) as region: + gpu_worker.pin_mmap_region(region) + base = region._base.data_ptr() + assert not region.is_pinned + assert region.pinned_addresses == [] + + registered = [base + i * 6 * PAGE_SIZE for i in range(fail_at)] + failed_size = min(6, 15 - 6 * fail_at) * PAGE_SIZE + assert cudart.mock_calls == [ + *(call.cudaHostRegister(address, 6 * PAGE_SIZE) for address in registered), + call.cudaHostRegister(base + fail_at * 6 * PAGE_SIZE, failed_size), + call.cudaGetLastError(), + *(call.cudaHostUnregister(address) for address in reversed(registered)), + ] + torch_cudart.cudaHostUnregister.assert_not_called() + + +def test_pin_mmap_region_without_cuda_runtime_stays_pageable( + iid, host_register, monkeypatch +): + """A CUDA runtime that cannot be loaded leaves the region pageable instead + of failing worker startup.""" + + def missing_runtime(): + raise AssertionError("libcudart is not loaded in the current process") + + monkeypatch.setattr(gpu_worker, "CudaRTLibrary", missing_runtime) + with _region(iid) as region: + gpu_worker.pin_mmap_region(region) + assert not region.is_pinned + assert region.pinned_addresses == [] + + def test_cleanup_after_create_next_worker_view_releases_mmap(iid): """cleanup() must close the mmap even after create_next_worker_view was called. create_next_worker_view returns a view that shares storage with _base; both must be diff --git a/vllm/distributed/device_communicators/cuda_wrapper.py b/vllm/distributed/device_communicators/cuda_wrapper.py index a5026163e9..d27d84c030 100644 --- a/vllm/distributed/device_communicators/cuda_wrapper.py +++ b/vllm/distributed/device_communicators/cuda_wrapper.py @@ -78,6 +78,16 @@ class CudaRTLibrary: cudaError_t, [ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint], ), + # cudaError_t cudaHostRegister ( void* ptr, size_t size, unsigned int flags ) + Function( + "cudaHostRegister", + cudaError_t, + [ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint], + ), + # cudaError_t cudaHostUnregister ( void* ptr ) + Function("cudaHostUnregister", cudaError_t, [ctypes.c_void_p]), + # cudaError_t cudaGetLastError ( void ) + Function("cudaGetLastError", cudaError_t, []), ] # https://rocm.docs.amd.com/projects/HIPIFY/en/latest/tables/CUDA_Runtime_API_functions_supported_by_HIP.html # noqa @@ -92,6 +102,9 @@ class CudaRTLibrary: "cudaMemcpy": "hipMemcpy", "cudaIpcGetMemHandle": "hipIpcGetMemHandle", "cudaIpcOpenMemHandle": "hipIpcOpenMemHandle", + "cudaHostRegister": "hipHostRegister", + "cudaHostUnregister": "hipHostUnregister", + "cudaGetLastError": "hipGetLastError", } # class attribute to store the mapping from the path to the library @@ -185,3 +198,16 @@ class CudaRTLibrary: ) ) return devPtr + + def cudaHostRegister(self, ptr: int, size: int, flags: int = 0) -> int: + """Return the raw error code instead of raising, since callers can + fall back to pageable memory. Drain a failure with cudaGetLastError.""" + return self.funcs["cudaHostRegister"](ctypes.c_void_p(ptr), size, flags) + + def cudaHostUnregister(self, ptr: int) -> int: + """Return the raw error code instead of raising.""" + return self.funcs["cudaHostUnregister"](ctypes.c_void_p(ptr)) + + def cudaGetLastError(self) -> int: + """Return and clear the error pending on this thread.""" + return self.funcs["cudaGetLastError"]() diff --git a/vllm/v1/kv_offload/cpu/gpu_worker.py b/vllm/v1/kv_offload/cpu/gpu_worker.py index 037e3631e4..797abed5ea 100644 --- a/vllm/v1/kv_offload/cpu/gpu_worker.py +++ b/vllm/v1/kv_offload/cpu/gpu_worker.py @@ -11,6 +11,7 @@ import numpy as np import torch from vllm import _custom_ops as ops +from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary from vllm.logger import init_logger from vllm.platforms import current_platform from vllm.triton_utils import HAS_TRITON, triton @@ -38,10 +39,13 @@ logger = init_logger(__name__) def _select_swap_blocks_fn( layer_refs_per_group: list[list[CanonicalKVCacheRef]], gpu_to_cpu: bool, + host_memory_is_pinned: bool = True, ): """Resolve the swap_blocks function for a handler at init time.""" # GPU->CPU is bandwidth-bound; the dedicated copy engine beats Triton. - if gpu_to_cpu: + # The Triton kernel dereferences CPU pointers on the GPU, which is only + # valid for pinned host memory. + if gpu_to_cpu or not host_memory_is_pinned: return ops.swap_blocks_batch # Fall back to the C++ DMA path on platforms where Triton isn't usable # (e.g. ROCm host mappings) or where GPU kernels cannot directly @@ -190,8 +194,12 @@ def _canonical_block_sizes( return canonical_bytes_per_block +# Bound registration size to avoid driver limits on large host allocations. +MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3 + + def pin_mmap_region(region: SharedOffloadRegion) -> None: - """Register the entire mmap as CUDA pinned memory via cudaHostRegister.""" + """Register row-aligned chunks, rolling back on failure.""" if not current_platform.is_cuda_alike(): logger.info( "Skipping mmap host registration on %s; cudaHostRegister is only " @@ -201,23 +209,65 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None: return rank = region.rank + try: + cudart = CudaRTLibrary() + except (AssertionError, AttributeError, OSError): + logger.warning( + "Could not load the CUDA runtime for host registration on rank=%d; " + "the offload region stays pageable", + rank, + exc_info=True, + ) + return base_ptr = region._base.data_ptr() - result = torch.cuda.cudart().cudaHostRegister(base_ptr, region.total_size_bytes, 0) - if result.value != 0: + total_size = region.total_size_bytes + # Chunks end on block-row boundaries, which are page aligned, so neither the + # driver's page rounding nor any single block transfer straddles two + # registrations. + rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1) + chunk_size = rows_per_chunk * region._row_stride + + # Register, drain and roll back through the same runtime handle, so a + # failed chunk leaves neither a pending error nor a partly pinned region. + addresses: list[int] = [] + for offset in range(0, total_size, chunk_size): + address = base_ptr + offset + size = min(chunk_size, total_size - offset) + result = cudart.cudaHostRegister(address, size) + if result == 0: + addresses.append(address) + continue + cudart.cudaGetLastError() logger.warning( - "cudaHostRegister failed for rank=%d (code=%d) — " - "transfers will still work but may be slower (unpinned DMA)", + "cudaHostRegister failed for rank=%d at %.2f of %.2f GB (code=%d); " + "the offload region stays pageable", rank, + offset / 1e9, + total_size / 1e9, result, ) - else: - logger.debug( - "cudaHostRegister rank=%d %.2f GB", - rank, - region.total_size_bytes / 1e9, - ) - region.is_pinned = True + for registered in reversed(addresses): + unregister_result = cudart.cudaHostUnregister(registered) + if unregister_result != 0: + cudart.cudaGetLastError() + logger.warning( + "cudaHostUnregister failed for rank=%d at %#x (code=%d); " + "that chunk stays registered until the process exits", + rank, + registered, + unregister_result, + ) + return + + region.pinned_addresses.extend(addresses) + region.is_pinned = True + logger.debug( + "cudaHostRegister rank=%d %.2f GB in %d chunk(s)", + rank, + total_size / 1e9, + len(addresses), + ) def _new_descriptor_buffers( @@ -248,6 +298,7 @@ class SingleDirectionOffloadingHandler: layer_refs_per_group: list[list[CanonicalKVCacheRef]], gpu_to_cpu: bool, canonical_layout: bool = False, + host_memory_is_pinned: bool = True, ): """Initialize a SingleDirectionOffloadingHandler. @@ -262,6 +313,8 @@ class SingleDirectionOffloadingHandler: gpu_to_cpu: if True, transfer from GPU to CPU; otherwise CPU to GPU. canonical_layout: if True, CPU pages use the canonical layout described by the refs' mappings. + host_memory_is_pinned: whether the CPU tensors are pinned, so GPU + kernels may dereference them directly. """ assert len(gpu_tensors) == len(cpu_tensors) @@ -299,7 +352,7 @@ class SingleDirectionOffloadingHandler: self.gpu_to_cpu: bool = gpu_to_cpu self.layer_refs_per_group = layer_refs_per_group self._swap_blocks_batch = _select_swap_blocks_fn( - layer_refs_per_group, gpu_to_cpu + layer_refs_per_group, gpu_to_cpu, host_memory_is_pinned ) # GPU blocks may be smaller @@ -646,7 +699,7 @@ class SingleDirectionOffloadingHandler: last_event = last_transfer.end_event # assure job will start only after the previous one completes stream.wait_event(last_event) - # CPU->GPU reads from host pinned memory, which is never written + # CPU->GPU reads from host memory, which is never written # by a concurrent GPU stream, so CU_MEMCPY_SRC_ACCESS_ORDER_ANY is # safe and lets the driver pipeline source reads. GPU->CPU reads # from the live GPU KV cache, which the compute stream keeps @@ -764,6 +817,9 @@ class CPUOffloadingWorker(OffloadingWorker): logger.info("Allocating %d CPU tensors...", len(kv_caches.tensors)) if mmap_region is not None and pin_memory: pin_mmap_region(mmap_region) + host_memory_is_pinned = pin_memory and ( + mmap_region is None or mmap_region.is_pinned + ) canonical_bytes_per_block = ( _canonical_block_sizes(kv_caches.group_data_refs, len(kv_caches.tensors)) @@ -813,6 +869,7 @@ class CPUOffloadingWorker(OffloadingWorker): layer_refs_per_group=kv_caches.group_data_refs, gpu_to_cpu=True, canonical_layout=canonical_layout, + host_memory_is_pinned=host_memory_is_pinned, ) self._load_handler = SingleDirectionOffloadingHandler( @@ -822,6 +879,7 @@ class CPUOffloadingWorker(OffloadingWorker): layer_refs_per_group=kv_caches.group_data_refs, gpu_to_cpu=False, canonical_layout=canonical_layout, + host_memory_is_pinned=host_memory_is_pinned, ) def submit_store(