[XPU] Support register KV offload mmap region as pinned host memory on XPU (#51956)
Signed-off-by: Chaojun Zhang <chaojun.zhang@intel.com>
This commit is contained in:
@@ -11,7 +11,6 @@ import numpy as np
|
||||
import torch
|
||||
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.distributed.device_communicators.cuda_wrapper import CudaRTLibrary
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import HAS_TRITON, triton
|
||||
@@ -27,6 +26,7 @@ from vllm.v1.kv_offload.base import (
|
||||
OffloadingWorker,
|
||||
TransferResult,
|
||||
)
|
||||
from vllm.v1.kv_offload.cpu.host_register import host_register, host_unregister
|
||||
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
|
||||
from vllm.v1.kv_offload.cpu.swap_blocks_triton import (
|
||||
THRESHOLD_BYTES,
|
||||
@@ -200,26 +200,7 @@ MAX_HOST_REGISTER_CHUNK_BYTES = 64 * 1024**3
|
||||
|
||||
def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
"""Register row-aligned chunks, rolling back on failure."""
|
||||
if not current_platform.is_cuda_alike():
|
||||
logger.info(
|
||||
"Skipping mmap host registration on %s; cudaHostRegister is only "
|
||||
"available on CUDA/ROCm.",
|
||||
current_platform.device_name,
|
||||
)
|
||||
return
|
||||
|
||||
rank = region.rank
|
||||
try:
|
||||
cudart = CudaRTLibrary()
|
||||
except (AssertionError, AttributeError, OSError):
|
||||
logger.warning(
|
||||
"Could not load the CUDA runtime for host registration on rank=%d; "
|
||||
"the offload region stays pageable",
|
||||
rank,
|
||||
exc_info=True,
|
||||
)
|
||||
return
|
||||
|
||||
base_ptr = region._base.data_ptr()
|
||||
total_size = region.total_size_bytes
|
||||
# Chunks end on block-row boundaries, which are page aligned, so neither the
|
||||
@@ -228,42 +209,30 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
|
||||
rows_per_chunk = max(MAX_HOST_REGISTER_CHUNK_BYTES // region._row_stride, 1)
|
||||
chunk_size = rows_per_chunk * region._row_stride
|
||||
|
||||
# Register, drain and roll back through the same runtime handle, so a
|
||||
# failed chunk leaves neither a pending error nor a partly pinned region.
|
||||
# Register, drain and roll back through the same helper, so a failed
|
||||
# chunk leaves neither a pending error nor a partly pinned region.
|
||||
addresses: list[int] = []
|
||||
for offset in range(0, total_size, chunk_size):
|
||||
address = base_ptr + offset
|
||||
size = min(chunk_size, total_size - offset)
|
||||
result = cudart.cudaHostRegister(address, size)
|
||||
if result == 0:
|
||||
if host_register(address, size):
|
||||
addresses.append(address)
|
||||
continue
|
||||
cudart.cudaGetLastError()
|
||||
logger.warning(
|
||||
"cudaHostRegister failed for rank=%d at %.2f of %.2f GB (code=%d); "
|
||||
"host_register failed for rank=%d at %.2f of %.2f GB; "
|
||||
"the offload region stays pageable",
|
||||
rank,
|
||||
offset / 1e9,
|
||||
total_size / 1e9,
|
||||
result,
|
||||
)
|
||||
for registered in reversed(addresses):
|
||||
unregister_result = cudart.cudaHostUnregister(registered)
|
||||
if unregister_result != 0:
|
||||
cudart.cudaGetLastError()
|
||||
logger.warning(
|
||||
"cudaHostUnregister failed for rank=%d at %#x (code=%d); "
|
||||
"that chunk stays registered until the process exits",
|
||||
rank,
|
||||
registered,
|
||||
unregister_result,
|
||||
)
|
||||
host_unregister(registered)
|
||||
return
|
||||
|
||||
region.pinned_addresses.extend(addresses)
|
||||
region.is_pinned = True
|
||||
logger.debug(
|
||||
"cudaHostRegister rank=%d %.2f GB in %d chunk(s)",
|
||||
"Host-registered mmap region rank=%d %.2f GB in %d chunk(s)",
|
||||
rank,
|
||||
total_size / 1e9,
|
||||
len(addresses),
|
||||
|
||||
Reference in New Issue
Block a user