[Bugfix][KV Offload] Register the offload region in chunks (#51081)
Signed-off-by: Mikhail Kostryukov <mike@triptrack.net> Co-authored-by: Claude <noreply@anthropic.com> Co-authored-by: Colton Ottley <coltonottley@gmail.com> Co-authored-by: Or Ozeri <oro@il.ibm.com>
This commit is contained in:
co-authored by
Claude
Colton Ottley
Or Ozeri
parent
08633cb5cd
commit
40b40d1d39
@@ -78,6 +78,16 @@ class CudaRTLibrary:
|
||||
cudaError_t,
|
||||
[ctypes.POINTER(ctypes.c_void_p), cudaIpcMemHandle_t, ctypes.c_uint],
|
||||
),
|
||||
# cudaError_t cudaHostRegister ( void* ptr, size_t size, unsigned int flags )
|
||||
Function(
|
||||
"cudaHostRegister",
|
||||
cudaError_t,
|
||||
[ctypes.c_void_p, ctypes.c_size_t, ctypes.c_uint],
|
||||
),
|
||||
# cudaError_t cudaHostUnregister ( void* ptr )
|
||||
Function("cudaHostUnregister", cudaError_t, [ctypes.c_void_p]),
|
||||
# cudaError_t cudaGetLastError ( void )
|
||||
Function("cudaGetLastError", cudaError_t, []),
|
||||
]
|
||||
|
||||
# https://rocm.docs.amd.com/projects/HIPIFY/en/latest/tables/CUDA_Runtime_API_functions_supported_by_HIP.html # noqa
|
||||
@@ -92,6 +102,9 @@ class CudaRTLibrary:
|
||||
"cudaMemcpy": "hipMemcpy",
|
||||
"cudaIpcGetMemHandle": "hipIpcGetMemHandle",
|
||||
"cudaIpcOpenMemHandle": "hipIpcOpenMemHandle",
|
||||
"cudaHostRegister": "hipHostRegister",
|
||||
"cudaHostUnregister": "hipHostUnregister",
|
||||
"cudaGetLastError": "hipGetLastError",
|
||||
}
|
||||
|
||||
# class attribute to store the mapping from the path to the library
|
||||
@@ -185,3 +198,16 @@ class CudaRTLibrary:
|
||||
)
|
||||
)
|
||||
return devPtr
|
||||
|
||||
def cudaHostRegister(self, ptr: int, size: int, flags: int = 0) -> int:
|
||||
"""Return the raw error code instead of raising, since callers can
|
||||
fall back to pageable memory. Drain a failure with cudaGetLastError."""
|
||||
return self.funcs["cudaHostRegister"](ctypes.c_void_p(ptr), size, flags)
|
||||
|
||||
def cudaHostUnregister(self, ptr: int) -> int:
|
||||
"""Return the raw error code instead of raising."""
|
||||
return self.funcs["cudaHostUnregister"](ctypes.c_void_p(ptr))
|
||||
|
||||
def cudaGetLastError(self) -> int:
|
||||
"""Return and clear the error pending on this thread."""
|
||||
return self.funcs["cudaGetLastError"]()
|
||||
|
||||
Reference in New Issue
Block a user