[ROCm] Enable SimpleCPUOffloadConnector on ROCm (#40549)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
Hongxia Yang
2026-05-06 20:52:02 -07:00
committed by GitHub
co-authored by TJian
parent 5a0a8fc1ea
commit 20cac26b19
4 changed files with 90 additions and 21 deletions
+27 -9
View File
@@ -97,13 +97,13 @@ void swap_blocks_batch(const torch::Tensor& src_ptrs,
const cudaStream_t stream = at::cuda::getCurrentCUDAStream();
// Use cuMemcpyBatchAsync (CUDA 12.8+) to submit all copies in a single
// driver call, amortizing per-copy submission overhead.
// int64_t and CUdeviceptr/size_t are both 8 bytes on 64-bit platforms,
// so we reinterpret_cast the tensor data directly to avoid copies.
static_assert(sizeof(CUdeviceptr) == sizeof(int64_t));
// Use cuMemcpyBatchAsync / hipMemcpyBatchAsync to submit all copies in a
// single driver call, amortizing per-copy submission overhead. int64_t
// and CUdeviceptr/void*/size_t are all 8 bytes on 64-bit platforms, so we
// reinterpret_cast the tensor data directly to avoid copies.
static_assert(sizeof(size_t) == sizeof(int64_t));
#if !defined(USE_ROCM) && defined(CUDA_VERSION) && CUDA_VERSION >= 12080
static_assert(sizeof(CUdeviceptr) == sizeof(int64_t));
// Resolve cuMemcpyBatchAsync at runtime via cuGetProcAddress so that
// binaries compiled with CUDA 12.8+ still work on older drivers, and
// we avoid the CUDA 13.0 header remapping (#define to _v2 signature).
@@ -134,12 +134,30 @@ void swap_blocks_batch(const torch::Tensor& src_ptrs,
&fail_idx, static_cast<CUstream>(stream));
TORCH_CHECK(result == CUDA_SUCCESS, "cuMemcpyBatchAsync failed at index ",
fail_idx, " with error ", result);
} else
return;
}
#elif defined(USE_ROCM) && defined(HIP_VERSION) && HIP_VERSION >= 70100000
// ROCm 7.1+ exposes hipMemcpyBatchAsync. The 7.2.1 implementation early-
// returns hipErrorNotSupported whenever numAttrs > 0 (see ROCm/clr @
// rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so call with
// numAttrs=0.
{
hipMemcpyAttributes attr = {};
size_t attrs_idx = 0;
size_t fail_idx = 0;
hipError_t result = hipMemcpyBatchAsync(
reinterpret_cast<void**>(dst_data), reinterpret_cast<void**>(src_data),
reinterpret_cast<size_t*>(size_data), static_cast<size_t>(n), &attr,
&attrs_idx, 0, &fail_idx, static_cast<hipStream_t>(stream));
TORCH_CHECK(result == hipSuccess, "hipMemcpyBatchAsync failed at index ",
fail_idx, " with error ", result);
return;
}
#endif
{
// Fallback for CUDA < 12.8, older drivers, and ROCm:
// individual async copies.
// cudaMemcpyDefault lets the driver infer direction from pointer types.
// Fallback for CUDA < 12.8, older CUDA drivers, and ROCm < 7.1:
// individual async copies. cudaMemcpyDefault lets the driver infer
// direction from pointer types.
for (int64_t i = 0; i < n; i++) {
cudaMemcpyAsync(reinterpret_cast<void*>(dst_data[i]),
reinterpret_cast<void*>(src_data[i]),