[ROCm] Enable SimpleCPUOffloadConnector on ROCm (#40549)

Signed-off-by: Hongxia Yang <hongxia.yang@amd.com>
Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
Hongxia Yang
2026-05-06 20:52:02 -07:00
committed by GitHub
co-authored by TJian
parent 5a0a8fc1ea
commit 20cac26b19
4 changed files with 90 additions and 21 deletions
+54 -9
View File
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Low-level CUDA memory helpers: pinning and batch DMA transfers."""
"""Low-level CUDA/HIP memory helpers: pinning and batch DMA transfers."""
import ctypes
from typing import Any, NamedTuple
@@ -9,6 +9,7 @@ import numpy as np
import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
logger = init_logger(__name__)
@@ -39,7 +40,7 @@ class _CUmemcpyAttributes(ctypes.Structure):
_BATCH_MEMCPY_FUNC_TYPE = ctypes.CFUNCTYPE(
ctypes.c_uint, # CUresult
ctypes.c_uint, # CUresult / hipError_t
ctypes.c_void_p,
ctypes.c_void_p,
ctypes.c_void_p,
@@ -56,7 +57,42 @@ _batch_memcpy_fn: Any = None
def _resolve_batch_memcpy():
"""Resolve cuMemcpyBatchAsync via cuGetProcAddress (one-time)."""
"""Resolve the platform batch-memcpy entry point (one-time).
* CUDA: ``cuMemcpyBatchAsync`` via ``cuGetProcAddress`` (uses
srcAccessOrder=STREAM via one attributes entry).
* ROCm: ``hipMemcpyBatchAsync`` from libamdhip64 (ROCm 7.1+). ROCm
7.2.1 or 7.2.2 rejects any call with ``numAttrs > 0``
(see ROCm/clr @ rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so
we call with ``numAttrs=0``.
Raises ``RuntimeError`` if the symbol is unavailable (older CUDA
driver, ROCm < 7.1, unusual install). The connector requires the
batch API.
"""
if current_platform.is_rocm():
try:
lib = ctypes.CDLL("libamdhip64.so", mode=ctypes.RTLD_GLOBAL)
fn = lib.hipMemcpyBatchAsync
except (OSError, AttributeError) as e:
raise RuntimeError(
"hipMemcpyBatchAsync is unavailable in this ROCm install; "
"SimpleCPUOffloadConnector requires ROCm 7.1+."
) from e
fn.restype = ctypes.c_uint
fn.argtypes = [
ctypes.c_void_p, # dsts
ctypes.c_void_p, # srcs
ctypes.c_void_p, # sizes
ctypes.c_size_t, # count
ctypes.c_void_p, # attrs
ctypes.c_void_p, # attrIdxs
ctypes.c_size_t, # numAttrs
ctypes.c_void_p, # failIdx
ctypes.c_void_p, # stream
]
return fn
from cuda.bindings import driver as drv
err, ptr, _ = drv.cuGetProcAddress(b"cuMemcpyBatchAsync", 12080, 0)
@@ -70,6 +106,8 @@ class BatchMemcpyParams(NamedTuple):
dst_bases: np.ndarray # [num_layers] uint64
bpb: np.ndarray # [num_layers] uint64 — bytes per block
num_layers: int
# CUDA only: one attributes entry with srcAccessOrder=ANY. Unused on
# ROCm (7.2.1 or 7.2.2) because the current runtime rejects numAttrs > 0.
attrs: _CUmemcpyAttributes
attrs_idx: ctypes.c_size_t
# NOTE: cuMemcpyBatchAsync_v2() removed fail_idx field, but we use
@@ -99,8 +137,10 @@ def build_params(
dst_bases.append(d.data_ptr())
bpb.append(s_bpb)
# Refer to https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6f1ff58e3065df3eb4b573dba77ad31f for details. # noqa: E501
attrs = _CUmemcpyAttributes(srcAccessOrder=3) # ANY
# ``srcAccessOrder=3`` == CU_MEMCPY_SRC_ACCESS_ORDER_ANY /
# hipMemcpySrcAccessOrderAny. See
# https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6f1ff58e3065df3eb4b573dba77ad31f # noqa: E501
attrs = _CUmemcpyAttributes(srcAccessOrder=3)
return BatchMemcpyParams(
src_bases=np.array(src_bases, dtype=np.uint64),
@@ -119,7 +159,7 @@ def copy_blocks(
dst_block_ids: list[int],
params: BatchMemcpyParams,
) -> None:
"""Copy blocks via cuMemcpyBatchAsync."""
"""Copy blocks via cuMemcpyBatchAsync / hipMemcpyBatchAsync."""
n = len(src_block_ids)
if n == 0:
return
@@ -134,8 +174,13 @@ def copy_blocks(
params.dst_bases[:, None] + dst_ids[None, :] * params.bpb[:, None]
).ravel()
sz_all = np.repeat(params.bpb, n)
total = n * params.num_layers
# ROCm 7.2.1/7.2.2 rejects any call with numAttrs>0 (hipMemcpyBatchAsync
# hipamd/src/hip_memory.cpp:2819-2822); CUDA uses one attrs entry so
# srcAccessOrder is honored. attrs / attrsIdxs are ignored when
# numAttrs==0, so we pass the same values from both paths.
num_attrs = 0 if current_platform.is_rocm() else 1
err = _batch_memcpy_fn(
dst_all.ctypes.data,
src_all.ctypes.data,
@@ -143,11 +188,11 @@ def copy_blocks(
total,
ctypes.addressof(params.attrs),
ctypes.byref(params.attrs_idx),
1,
num_attrs,
ctypes.byref(params.fail_idx),
params.stream_handle,
)
if err != 0:
raise RuntimeError(
f"cuMemcpyBatchAsync failed: err={err} failIdx={params.fail_idx.value}"
f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}"
)