[ROCm] Cpu offload for ROCm 7.13+ to align the hipMemcpyBatchAsync params and perf in 7.14x (#43018)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
c1d7a3808d
commit
cd7b7c265a
@@ -299,6 +299,7 @@ if TYPE_CHECKING:
|
||||
VLLM_DEBUG_MFU_METRICS: bool = False
|
||||
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
||||
VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False
|
||||
VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS: int = 0
|
||||
VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False
|
||||
VLLM_DISABLE_LOG_LOGO: bool = False
|
||||
VLLM_LORA_DISABLE_PDL: bool = False
|
||||
@@ -2059,6 +2060,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool(
|
||||
int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0"))
|
||||
),
|
||||
# Max descriptors per CPU-KV-offload batch-memcpy call. 0 = platform default
|
||||
# (ROCm chunks at 8192, since hipMemcpyBatchAsync faults above that on
|
||||
# rocm 7.14/7.15 when batch copy is optimized; CUDA uncapped). Set >0 to override.
|
||||
"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS": lambda: int(
|
||||
os.getenv("VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS", "0")
|
||||
),
|
||||
# On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is
|
||||
# supported but disabled by default due to a small performance regression.
|
||||
# Set to 1 when pinned memory or UVA is required (e.g. CPU offloading
|
||||
|
||||
@@ -8,6 +8,7 @@ from typing import Any, NamedTuple
|
||||
import numpy as np
|
||||
import torch
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -58,27 +59,47 @@ _BATCH_MEMCPY_FUNC_TYPE = ctypes.CFUNCTYPE(
|
||||
ctypes.c_void_p,
|
||||
)
|
||||
|
||||
# Resolved lazily on first use.
|
||||
_batch_memcpy_fn: Any = None
|
||||
# Resolved lazily on first use: (entry point, numAttrs to pass).
|
||||
_batch_memcpy: tuple[Any, int] | None = None
|
||||
|
||||
# Max copy descriptors per batch call, resolved lazily on first use.
|
||||
_max_batch_descriptors: int | None = None
|
||||
|
||||
# ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so we chunk
|
||||
# at that ceiling. Override via VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS.
|
||||
_ROCM_DEFAULT_MAX_BATCH_DESCRIPTORS = 8192
|
||||
|
||||
|
||||
def _resolve_batch_memcpy():
|
||||
"""Resolve the platform batch-memcpy entry point (one-time).
|
||||
def _resolve_max_batch_descriptors() -> int:
|
||||
"""Max copy descriptors to pass to one batch-memcpy call (0 = unlimited).
|
||||
|
||||
* CUDA: ``cuMemcpyBatchAsync`` via ``cuGetProcAddress`` (uses
|
||||
srcAccessOrder=STREAM via one attributes entry).
|
||||
* ROCm: ``hipMemcpyBatchAsync`` from libamdhip64 (ROCm 7.1+). ROCm
|
||||
7.2.1 or 7.2.2 rejects any call with ``numAttrs > 0``
|
||||
(see ROCm/clr @ rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so
|
||||
we call with ``numAttrs=0``.
|
||||
ROCm's ``hipMemcpyBatchAsync`` faults above 8192 descriptors per call, so
|
||||
on ROCm we cap and chunk larger transfers. CUDA's ``cuMemcpyBatchAsync``
|
||||
handles arbitrary counts and is left uncapped. Set
|
||||
``VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS`` (>0) to override on any platform.
|
||||
"""
|
||||
global _max_batch_descriptors
|
||||
if _max_batch_descriptors is None:
|
||||
override = envs.VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS
|
||||
if override > 0:
|
||||
_max_batch_descriptors = override
|
||||
else:
|
||||
_max_batch_descriptors = (
|
||||
_ROCM_DEFAULT_MAX_BATCH_DESCRIPTORS if current_platform.is_rocm() else 0
|
||||
)
|
||||
return _max_batch_descriptors
|
||||
|
||||
Raises ``RuntimeError`` if the symbol is unavailable (older CUDA
|
||||
driver, ROCm < 7.1, unusual install). The connector requires the
|
||||
batch API.
|
||||
|
||||
def _resolve_batch_memcpy() -> tuple[Any, int]:
|
||||
"""Resolve the batch-memcpy entry point and its ``numAttrs`` (one-time).
|
||||
|
||||
CUDA uses ``cuMemcpyBatchAsync``; ROCm uses ``hipMemcpyBatchAsync``.
|
||||
Raises ``RuntimeError`` if the symbol is unavailable (old CUDA driver,
|
||||
ROCm < 7.1, unusual install).
|
||||
"""
|
||||
if current_platform.is_rocm():
|
||||
try:
|
||||
lib = ctypes.CDLL("libamdhip64.so", mode=ctypes.RTLD_GLOBAL)
|
||||
lib = _load_hip_runtime()
|
||||
fn = lib.hipMemcpyBatchAsync
|
||||
except (OSError, AttributeError) as e:
|
||||
raise RuntimeError(
|
||||
@@ -97,14 +118,55 @@ def _resolve_batch_memcpy():
|
||||
ctypes.c_void_p, # failIdx
|
||||
ctypes.c_void_p, # stream
|
||||
]
|
||||
return fn
|
||||
return fn, _rocm_num_attrs(lib)
|
||||
|
||||
from cuda.bindings import driver as drv
|
||||
|
||||
err, ptr, _ = drv.cuGetProcAddress(b"cuMemcpyBatchAsync", 12080, 0)
|
||||
if err != drv.CUresult.CUDA_SUCCESS:
|
||||
raise RuntimeError(f"cuGetProcAddress(cuMemcpyBatchAsync) failed: {err}")
|
||||
return _BATCH_MEMCPY_FUNC_TYPE(ptr)
|
||||
return _BATCH_MEMCPY_FUNC_TYPE(ptr), 1
|
||||
|
||||
|
||||
def _load_hip_runtime() -> ctypes.CDLL:
|
||||
"""Load ``libamdhip64``, tolerating installs without the devel symlink.
|
||||
|
||||
The unversioned ``libamdhip64.so`` only ships with the ROCm devel package;
|
||||
runtime-only and wheel-packaged ROCm installs provide just the versioned
|
||||
soname. ``dlopen`` returns the already-mapped library when asked for a
|
||||
soname the process has loaded — torch loads HIP at import — so the
|
||||
versioned names resolve even when they are not on the loader search path.
|
||||
"""
|
||||
errors = []
|
||||
for name in ("libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.6"):
|
||||
try:
|
||||
return ctypes.CDLL(name, mode=ctypes.RTLD_GLOBAL)
|
||||
except OSError as e:
|
||||
errors.append(f"{name}: {e}")
|
||||
raise OSError("could not load the HIP runtime: " + "; ".join(errors))
|
||||
|
||||
|
||||
def _rocm_num_attrs(lib: ctypes.CDLL) -> int:
|
||||
"""``numAttrs`` for ``hipMemcpyBatchAsync`` on the running HIP runtime."""
|
||||
ver = ctypes.c_int(0)
|
||||
try:
|
||||
if lib.hipRuntimeGetVersion(ctypes.byref(ver)) != 0:
|
||||
ver.value = 0
|
||||
except (OSError, AttributeError):
|
||||
ver.value = 0
|
||||
return _num_attrs_for_hip_version(ver.value)
|
||||
|
||||
|
||||
def _num_attrs_for_hip_version(version: int) -> int:
|
||||
"""``numAttrs`` for ``hipMemcpyBatchAsync`` given a HIP runtime version int.
|
||||
|
||||
ROCm 7.2.1-7.2.3 reject ``numAttrs > 0`` (ROCm/clr @ rocm-7.2.1
|
||||
hipamd/src/hip_memory.cpp:2819-2822); 7.13+ accept it. ``version`` 0
|
||||
(unknown) yields the conservative 0.
|
||||
"""
|
||||
# HIP encodes version as major*10_000_000 + minor*100_000 + patch.
|
||||
major, minor = version // 10_000_000, (version // 100_000) % 100
|
||||
return 1 if (major, minor) >= (7, 13) else 0
|
||||
|
||||
|
||||
class BatchMemcpyParams(NamedTuple):
|
||||
@@ -112,10 +174,12 @@ class BatchMemcpyParams(NamedTuple):
|
||||
dst_bases: np.ndarray # [num_layers] uint64
|
||||
bpb: np.ndarray # [num_layers] uint64 — bytes per block
|
||||
num_layers: int
|
||||
# CUDA only: one attributes entry carrying srcAccessOrder. Unused on ROCm
|
||||
# (7.2.1 or 7.2.2) because the current runtime rejects numAttrs > 0.
|
||||
# One attributes entry carrying srcAccessOrder. Ignored when num_attrs is
|
||||
# 0, which is what ROCm runtimes older than 7.13 require (see
|
||||
# _num_attrs_for_hip_version).
|
||||
attrs: _CUmemcpyAttributes
|
||||
attrs_idx: ctypes.c_size_t
|
||||
num_attrs: int
|
||||
# NOTE: cuMemcpyBatchAsync_v2() removed fail_idx field, but we use
|
||||
# cuMemcpyBatchAsync() with fail_idx for backward compatibility
|
||||
fail_idx: ctypes.c_size_t
|
||||
@@ -128,9 +192,10 @@ def build_params(
|
||||
stream: torch.cuda.Stream,
|
||||
src_access_order: int = CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
|
||||
) -> BatchMemcpyParams:
|
||||
global _batch_memcpy_fn
|
||||
if _batch_memcpy_fn is None:
|
||||
_batch_memcpy_fn = _resolve_batch_memcpy()
|
||||
global _batch_memcpy
|
||||
if _batch_memcpy is None:
|
||||
_batch_memcpy = _resolve_batch_memcpy()
|
||||
_, num_attrs = _batch_memcpy
|
||||
|
||||
assert list(src_caches.keys()) == list(dst_caches.keys())
|
||||
src_tensors = list(src_caches.values())
|
||||
@@ -153,6 +218,7 @@ def build_params(
|
||||
num_layers=len(src_tensors),
|
||||
attrs=attrs,
|
||||
attrs_idx=ctypes.c_size_t(0),
|
||||
num_attrs=num_attrs,
|
||||
fail_idx=ctypes.c_size_t(0),
|
||||
stream_handle=stream.cuda_stream,
|
||||
)
|
||||
@@ -168,6 +234,9 @@ def copy_blocks(
|
||||
if n == 0:
|
||||
return
|
||||
|
||||
assert _batch_memcpy is not None, "build_params() must run before copy_blocks()"
|
||||
fn, _ = _batch_memcpy
|
||||
|
||||
src_ids = np.array(src_block_ids, dtype=np.uint64)
|
||||
dst_ids = np.array(dst_block_ids, dtype=np.uint64)
|
||||
|
||||
@@ -180,23 +249,24 @@ def copy_blocks(
|
||||
sz_all = np.repeat(params.bpb, n)
|
||||
total = n * params.num_layers
|
||||
|
||||
# ROCm 7.2.1/7.2.2 rejects any call with numAttrs>0 (hipMemcpyBatchAsync
|
||||
# hipamd/src/hip_memory.cpp:2819-2822); CUDA uses one attrs entry so
|
||||
# srcAccessOrder is honored. attrs / attrsIdxs are ignored when
|
||||
# numAttrs==0, so we pass the same values from both paths.
|
||||
num_attrs = 0 if current_platform.is_rocm() else 1
|
||||
err = _batch_memcpy_fn(
|
||||
dst_all.ctypes.data,
|
||||
src_all.ctypes.data,
|
||||
sz_all.ctypes.data,
|
||||
total,
|
||||
ctypes.addressof(params.attrs),
|
||||
ctypes.byref(params.attrs_idx),
|
||||
num_attrs,
|
||||
ctypes.byref(params.fail_idx),
|
||||
params.stream_handle,
|
||||
)
|
||||
if err != 0:
|
||||
raise RuntimeError(
|
||||
f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}"
|
||||
# Chunk on ROCm: hipMemcpyBatchAsync faults above 8192 descriptors/call.
|
||||
# CUDA is uncapped (max_desc == 0) and issues a single call.
|
||||
max_desc = _resolve_max_batch_descriptors()
|
||||
step = total if max_desc <= 0 else max_desc
|
||||
for off in range(0, total, step):
|
||||
cnt = min(step, total - off)
|
||||
err = fn(
|
||||
dst_all[off : off + cnt].ctypes.data,
|
||||
src_all[off : off + cnt].ctypes.data,
|
||||
sz_all[off : off + cnt].ctypes.data,
|
||||
cnt,
|
||||
ctypes.addressof(params.attrs),
|
||||
ctypes.byref(params.attrs_idx),
|
||||
params.num_attrs,
|
||||
ctypes.byref(params.fail_idx),
|
||||
params.stream_handle,
|
||||
)
|
||||
if err != 0:
|
||||
raise RuntimeError(
|
||||
f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}"
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user