[Bugfix][KV Offload] Register the offload region in chunks (#51081)

Signed-off-by: Mikhail Kostryukov <mike@triptrack.net>
Co-authored-by: Claude <noreply@anthropic.com>
Co-authored-by: Colton Ottley <coltonottley@gmail.com>
Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
drakosha
2026-09-17 13:19:56 +03:00
committed by GitHub
co-authored by Claude Colton Ottley Or Ozeri
parent 08633cb5cd
commit 40b40d1d39
5 changed files with 198 additions and 15 deletions
@@ -78,6 +78,16 @@ class CudaRTLibrary:
cudaError_t,
[ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint],
),
# cudaError_t cudaHostRegister ( void* ptr, size_t size, unsigned int flags )
Function(
"cudaHostRegister",
cudaError_t,
[ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint],
),
# cudaError_t cudaHostUnregister ( void* ptr )
Function("cudaHostUnregister", cudaError_t, [ctypes.c_void_p]),
# cudaError_t cudaGetLastError ( void )
Function("cudaGetLastError", cudaError_t, []),
]
# https://rocm.docs.amd.com/projects/HIPIFY/en/latest/tables/CUDA_Runtime_API_functions_supported_by_HIP.html # noqa
@@ -92,6 +102,9 @@ class CudaRTLibrary:
"cudaMemcpy": "hipMemcpy",
"cudaIpcGetMemHandle": "hipIpcGetMemHandle",
"cudaIpcOpenMemHandle": "hipIpcOpenMemHandle",
"cudaHostRegister": "hipHostRegister",
"cudaHostUnregister": "hipHostUnregister",
"cudaGetLastError": "hipGetLastError",
}
# class attribute to store the mapping from the path to the library
@@ -185,3 +198,16 @@ class CudaRTLibrary:
)
)
return devPtr
def cudaHostRegister(self, ptr: int, size: int, flags: int = 0) -> int:
"""Return the raw error code instead of raising, since callers can
fall back to pageable memory. Drain a failure with cudaGetLastError."""
return self.funcs["cudaHostRegister"](ctypes.c_void_p(ptr), size, flags)
def cudaHostUnregister(self, ptr: int) -> int:
"""Return the raw error code instead of raising."""
return self.funcs["cudaHostUnregister"](ctypes.c_void_p(ptr))
def cudaGetLastError(self) -> int:
"""Return and clear the error pending on this thread."""
return self.funcs["cudaGetLastError"]()