[ROCm] Cpu offload for ROCm 7.13+ to align the hipMemcpyBatchAsync params and perf in 7.14x (#43018)
Signed-off-by: Hongxia Yang <hongxia.yang@amd.com> Co-authored-by: Yifan Qiao <yifanqiao@inferact.ai>
This commit is contained in:
co-authored by
Yifan Qiao
parent
c1d7a3808d
commit
cd7b7c265a
@@ -16,6 +16,7 @@
|
|||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cassert>
|
#include <cassert>
|
||||||
#include <cfloat>
|
#include <cfloat>
|
||||||
|
#include <cstdlib>
|
||||||
|
|
||||||
#ifdef USE_ROCM
|
#ifdef USE_ROCM
|
||||||
#include <hip/hip_bf16.h>
|
#include <hip/hip_bf16.h>
|
||||||
@@ -76,6 +77,31 @@ void swap_blocks(torch::stable::Tensor& src, torch::stable::Tensor& dst,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
// ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so chunk
|
||||||
|
// at that ceiling.
|
||||||
|
constexpr int64_t kRocmDefaultMaxBatchDescriptors = 8192;
|
||||||
|
|
||||||
|
constexpr const char* kMaxBatchDescriptorsEnv =
|
||||||
|
"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS";
|
||||||
|
|
||||||
|
// Max copy descriptors per batch-memcpy call (0 = unlimited). ROCm caps and
|
||||||
|
// chunks; CUDA is uncapped. The env var (>0) overrides on any platform.
|
||||||
|
int64_t resolve_max_batch_descriptors() {
|
||||||
|
static const int64_t cached = []() -> int64_t {
|
||||||
|
const char* val = std::getenv(kMaxBatchDescriptorsEnv);
|
||||||
|
const int64_t override_val = val ? std::atoll(val) : 0;
|
||||||
|
if (override_val > 0) return override_val;
|
||||||
|
#if defined(USE_ROCM)
|
||||||
|
return kRocmDefaultMaxBatchDescriptors;
|
||||||
|
#else
|
||||||
|
return 0;
|
||||||
|
#endif
|
||||||
|
}();
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
} // namespace
|
||||||
|
|
||||||
void swap_blocks_batch(const torch::stable::Tensor& src_ptrs,
|
void swap_blocks_batch(const torch::stable::Tensor& src_ptrs,
|
||||||
const torch::stable::Tensor& dst_ptrs,
|
const torch::stable::Tensor& dst_ptrs,
|
||||||
const torch::stable::Tensor& sizes,
|
const torch::stable::Tensor& sizes,
|
||||||
@@ -142,32 +168,63 @@ void swap_blocks_batch(const torch::stable::Tensor& src_ptrs,
|
|||||||
: CU_MEMCPY_SRC_ACCESS_ORDER_STREAM;
|
: CU_MEMCPY_SRC_ACCESS_ORDER_STREAM;
|
||||||
size_t attrs_idx = 0;
|
size_t attrs_idx = 0;
|
||||||
size_t fail_idx = 0;
|
size_t fail_idx = 0;
|
||||||
CUresult result = batch_fn(reinterpret_cast<CUdeviceptr*>(dst_data),
|
// Uncapped on CUDA (max_desc == 0 -> single call) unless overridden by
|
||||||
reinterpret_cast<CUdeviceptr*>(src_data),
|
// VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS; chunk to honor the override.
|
||||||
reinterpret_cast<size_t*>(size_data),
|
const int64_t max_desc = resolve_max_batch_descriptors();
|
||||||
static_cast<size_t>(n), &attr, &attrs_idx, 1,
|
const int64_t step = max_desc <= 0 ? n : max_desc;
|
||||||
&fail_idx, static_cast<CUstream>(stream));
|
for (int64_t off = 0; off < n; off += step) {
|
||||||
STD_TORCH_CHECK(result == CUDA_SUCCESS,
|
const int64_t cnt = std::min(step, n - off);
|
||||||
"cuMemcpyBatchAsync failed at index ", fail_idx,
|
CUresult result = batch_fn(reinterpret_cast<CUdeviceptr*>(dst_data + off),
|
||||||
" with error ", result);
|
reinterpret_cast<CUdeviceptr*>(src_data + off),
|
||||||
|
reinterpret_cast<size_t*>(size_data + off),
|
||||||
|
static_cast<size_t>(cnt), &attr, &attrs_idx, 1,
|
||||||
|
&fail_idx, static_cast<CUstream>(stream));
|
||||||
|
STD_TORCH_CHECK(result == CUDA_SUCCESS,
|
||||||
|
"cuMemcpyBatchAsync failed at index ", fail_idx,
|
||||||
|
" with error ", result);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
#elif defined(USE_ROCM) && defined(HIP_VERSION) && HIP_VERSION >= 70100000
|
#elif defined(USE_ROCM) && defined(HIP_VERSION) && HIP_VERSION >= 70100000
|
||||||
// ROCm 7.1+ exposes hipMemcpyBatchAsync. The 7.2.1 implementation early-
|
// ROCm 7.1+ exposes hipMemcpyBatchAsync. ROCm 7.2.1-7.2.3 early-return
|
||||||
// returns hipErrorNotSupported whenever numAttrs > 0 (see ROCm/clr @
|
// hipErrorNotSupported whenever numAttrs > 0 (see ROCm/clr @ rocm-7.2.1
|
||||||
// rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so call with
|
// hipamd/src/hip_memory.cpp:2819-2822), so those releases must call with
|
||||||
// numAttrs=0.
|
// numAttrs=0. ROCm 7.13+ accepts numAttrs > 0.
|
||||||
|
// rocm-7.14+ has better performance.
|
||||||
{
|
{
|
||||||
hipMemcpyAttributes attr = {};
|
hipMemcpyAttributes attr = {};
|
||||||
size_t attrs_idx = 0;
|
size_t attrs_idx = 0;
|
||||||
size_t fail_idx = 0;
|
size_t fail_idx = 0;
|
||||||
hipError_t result = hipMemcpyBatchAsync(
|
size_t num_attrs = 0;
|
||||||
reinterpret_cast<void**>(dst_data), reinterpret_cast<void**>(src_data),
|
#if HIP_VERSION >= 71300000
|
||||||
reinterpret_cast<size_t*>(size_data), static_cast<size_t>(n), &attr,
|
static const bool runtime_accepts_attrs = []() {
|
||||||
&attrs_idx, 0, &fail_idx, static_cast<hipStream_t>(stream));
|
int runtime_version = 0;
|
||||||
STD_TORCH_CHECK(result == hipSuccess,
|
return hipRuntimeGetVersion(&runtime_version) == hipSuccess &&
|
||||||
"hipMemcpyBatchAsync failed at index ", fail_idx,
|
runtime_version >= 71300000;
|
||||||
" with error ", result);
|
}();
|
||||||
|
if (runtime_accepts_attrs) {
|
||||||
|
attr.srcAccessOrder = is_src_access_order_any
|
||||||
|
? hipMemcpySrcAccessOrderAny
|
||||||
|
: hipMemcpySrcAccessOrderStream;
|
||||||
|
num_attrs = 1;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
// hipMemcpyBatchAsync faults (GPU memory access fault) above 8192
|
||||||
|
// descriptors/call on ROCm 7.15, so chunk the batch at the resolved cap.
|
||||||
|
const int64_t max_desc = resolve_max_batch_descriptors();
|
||||||
|
const int64_t step = max_desc <= 0 ? n : max_desc;
|
||||||
|
for (int64_t off = 0; off < n; off += step) {
|
||||||
|
const int64_t cnt = std::min(step, n - off);
|
||||||
|
hipError_t result = hipMemcpyBatchAsync(
|
||||||
|
reinterpret_cast<void**>(dst_data + off),
|
||||||
|
reinterpret_cast<void**>(src_data + off),
|
||||||
|
reinterpret_cast<size_t*>(size_data + off), static_cast<size_t>(cnt),
|
||||||
|
&attr, &attrs_idx, num_attrs, &fail_idx,
|
||||||
|
static_cast<hipStream_t>(stream));
|
||||||
|
STD_TORCH_CHECK(result == hipSuccess,
|
||||||
|
"hipMemcpyBatchAsync failed at index ", fail_idx,
|
||||||
|
" with error ", result);
|
||||||
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|||||||
@@ -0,0 +1,145 @@
|
|||||||
|
# SPDX-License-Identifier: Apache-2.0
|
||||||
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
"""Unit tests for the HIP/ROCm paths in vllm.v1.simple_kv_offload.cuda_mem_ops.
|
||||||
|
|
||||||
|
Deliberately not skipped off ROCm: the version gate takes a plain version int
|
||||||
|
and the library loader is monkeypatched, so nothing here needs a HIP runtime
|
||||||
|
and the logic stays covered by the default (NVIDIA) CI that runs this
|
||||||
|
directory.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import ctypes
|
||||||
|
|
||||||
|
import numpy as np
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
from vllm.v1.simple_kv_offload import cuda_mem_ops
|
||||||
|
from vllm.v1.simple_kv_offload.cuda_mem_ops import (
|
||||||
|
BatchMemcpyParams,
|
||||||
|
_CUmemcpyAttributes,
|
||||||
|
_load_hip_runtime,
|
||||||
|
_num_attrs_for_hip_version,
|
||||||
|
copy_blocks,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# hipMemcpyBatchAsync rejects numAttrs > 0 on ROCm 7.2.1-7.2.3 and accepts it
|
||||||
|
# on 7.13+. HIP encodes the version as major*10_000_000 + minor*100_000 + patch.
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("version", "expected"),
|
||||||
|
[
|
||||||
|
(0, 0), # unknown (hipRuntimeGetVersion failed) -> conservative 0
|
||||||
|
(70_100_000, 0), # ROCm 7.1
|
||||||
|
(70_200_000, 0), # ROCm 7.2.0
|
||||||
|
(70_253_211, 0), # ROCm 7.2.2 (patch 53211)
|
||||||
|
(70_300_000, 0), # ROCm 7.3
|
||||||
|
(71_200_000, 0), # ROCm 7.12 -- just below the 7.13 cutoff
|
||||||
|
(71_300_000, 1), # ROCm 7.13 -- first release accepting numAttrs > 0
|
||||||
|
(71_326_193, 1), # ROCm 7.13 nightly (patch 26193)
|
||||||
|
(71_400_000, 1), # ROCm 7.14
|
||||||
|
(71_500_000, 1), # ROCm 7.15
|
||||||
|
(80_000_000, 1), # ROCm 8.0
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_num_attrs_for_hip_version(version: int, expected: int):
|
||||||
|
assert _num_attrs_for_hip_version(version) == expected
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("available", "expected"),
|
||||||
|
[
|
||||||
|
# Devel install: the unversioned symlink is present and preferred.
|
||||||
|
({"libamdhip64.so", "libamdhip64.so.7"}, "libamdhip64.so"),
|
||||||
|
# Runtime-only / wheel-packaged ROCm: versioned soname only.
|
||||||
|
({"libamdhip64.so.7"}, "libamdhip64.so.7"),
|
||||||
|
({"libamdhip64.so.6"}, "libamdhip64.so.6"),
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_load_hip_runtime_falls_back_to_versioned_soname(
|
||||||
|
monkeypatch, available: set[str], expected: str
|
||||||
|
):
|
||||||
|
loaded = []
|
||||||
|
|
||||||
|
def fake_cdll(name, mode=0):
|
||||||
|
if name not in available:
|
||||||
|
raise OSError(f"{name}: cannot open shared object file")
|
||||||
|
loaded.append(name)
|
||||||
|
return name
|
||||||
|
|
||||||
|
monkeypatch.setattr(cuda_mem_ops.ctypes, "CDLL", fake_cdll)
|
||||||
|
assert _load_hip_runtime() == expected
|
||||||
|
assert loaded == [expected]
|
||||||
|
|
||||||
|
|
||||||
|
def test_load_hip_runtime_reports_every_candidate_when_all_fail(monkeypatch):
|
||||||
|
def fake_cdll(name, mode=0):
|
||||||
|
raise OSError("cannot open shared object file")
|
||||||
|
|
||||||
|
monkeypatch.setattr(cuda_mem_ops.ctypes, "CDLL", fake_cdll)
|
||||||
|
with pytest.raises(OSError, match="could not load the HIP runtime") as exc:
|
||||||
|
_load_hip_runtime()
|
||||||
|
for name in ("libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.6"):
|
||||||
|
assert name in str(exc.value)
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_params(num_layers: int, bytes_per_block: int = 512) -> BatchMemcpyParams:
|
||||||
|
"""Minimal params for driving copy_blocks() without a real device.
|
||||||
|
|
||||||
|
Bases are 0 and the batch-memcpy function is monkeypatched, so no memory is
|
||||||
|
ever dereferenced; only the descriptor bookkeeping (count/chunking) matters.
|
||||||
|
"""
|
||||||
|
return BatchMemcpyParams(
|
||||||
|
src_bases=np.zeros(num_layers, dtype=np.uint64),
|
||||||
|
dst_bases=np.zeros(num_layers, dtype=np.uint64),
|
||||||
|
bpb=np.full(num_layers, bytes_per_block, dtype=np.uint64),
|
||||||
|
num_layers=num_layers,
|
||||||
|
attrs=_CUmemcpyAttributes(),
|
||||||
|
attrs_idx=ctypes.c_size_t(0),
|
||||||
|
num_attrs=0,
|
||||||
|
fail_idx=ctypes.c_size_t(0),
|
||||||
|
stream_handle=0,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _record_counts(monkeypatch) -> list[int]:
|
||||||
|
"""Patch the batch-memcpy call to record the descriptor count per call."""
|
||||||
|
counts: list[int] = []
|
||||||
|
|
||||||
|
def fake_fn(dst, src, sizes, count, *rest):
|
||||||
|
counts.append(count)
|
||||||
|
return 0
|
||||||
|
|
||||||
|
monkeypatch.setattr(cuda_mem_ops, "_batch_memcpy", (fake_fn, 0))
|
||||||
|
return counts
|
||||||
|
|
||||||
|
|
||||||
|
# ROCm chunks the descriptor batch at the cap (hipMemcpyBatchAsync faults above
|
||||||
|
# 8192/call); CUDA leaves the cap at 0 and issues a single call.
|
||||||
|
@pytest.mark.parametrize(
|
||||||
|
("max_desc", "num_layers", "num_blocks", "expected_counts"),
|
||||||
|
[
|
||||||
|
(8192, 3, 5000, [8192, 6808]), # 15000 desc -> 8192 + 6808
|
||||||
|
(8192, 1, 8192, [8192]), # exactly at the cap -> single call
|
||||||
|
(8192, 1, 8193, [8192, 1]), # one over -> spills into a second call
|
||||||
|
(4096, 61, 135, [4096, 4096, 43]), # bs=1 Kimi shape (8235 desc)
|
||||||
|
(0, 61, 135, [8235]), # CUDA uncapped -> single call
|
||||||
|
],
|
||||||
|
)
|
||||||
|
def test_copy_blocks_chunks_at_descriptor_cap(
|
||||||
|
monkeypatch, max_desc, num_layers, num_blocks, expected_counts
|
||||||
|
):
|
||||||
|
counts = _record_counts(monkeypatch)
|
||||||
|
monkeypatch.setattr(cuda_mem_ops, "_max_batch_descriptors", max_desc)
|
||||||
|
|
||||||
|
ids = list(range(num_blocks))
|
||||||
|
copy_blocks(ids, ids, _fake_params(num_layers))
|
||||||
|
|
||||||
|
assert counts == expected_counts
|
||||||
|
assert sum(counts) == num_layers * num_blocks
|
||||||
|
|
||||||
|
|
||||||
|
def test_copy_blocks_noop_on_empty(monkeypatch):
|
||||||
|
counts = _record_counts(monkeypatch)
|
||||||
|
monkeypatch.setattr(cuda_mem_ops, "_max_batch_descriptors", 8192)
|
||||||
|
copy_blocks([], [], _fake_params(num_layers=4))
|
||||||
|
assert counts == []
|
||||||
@@ -299,6 +299,7 @@ if TYPE_CHECKING:
|
|||||||
VLLM_DEBUG_MFU_METRICS: bool = False
|
VLLM_DEBUG_MFU_METRICS: bool = False
|
||||||
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
|
||||||
VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False
|
VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False
|
||||||
|
VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS: int = 0
|
||||||
VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False
|
VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False
|
||||||
VLLM_DISABLE_LOG_LOGO: bool = False
|
VLLM_DISABLE_LOG_LOGO: bool = False
|
||||||
VLLM_LORA_DISABLE_PDL: bool = False
|
VLLM_LORA_DISABLE_PDL: bool = False
|
||||||
@@ -2059,6 +2060,12 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
|||||||
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool(
|
"VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool(
|
||||||
int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0"))
|
int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0"))
|
||||||
),
|
),
|
||||||
|
# Max descriptors per CPU-KV-offload batch-memcpy call. 0 = platform default
|
||||||
|
# (ROCm chunks at 8192, since hipMemcpyBatchAsync faults above that on
|
||||||
|
# rocm 7.14/7.15 when batch copy is optimized; CUDA uncapped). Set >0 to override.
|
||||||
|
"VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS": lambda: int(
|
||||||
|
os.getenv("VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS", "0")
|
||||||
|
),
|
||||||
# On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is
|
# On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is
|
||||||
# supported but disabled by default due to a small performance regression.
|
# supported but disabled by default due to a small performance regression.
|
||||||
# Set to 1 when pinned memory or UVA is required (e.g. CPU offloading
|
# Set to 1 when pinned memory or UVA is required (e.g. CPU offloading
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ from typing import Any, NamedTuple
|
|||||||
import numpy as np
|
import numpy as np
|
||||||
import torch
|
import torch
|
||||||
|
|
||||||
|
import vllm.envs as envs
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.platforms import current_platform
|
from vllm.platforms import current_platform
|
||||||
|
|
||||||
@@ -58,27 +59,47 @@ _BATCH_MEMCPY_FUNC_TYPE = ctypes.CFUNCTYPE(
|
|||||||
ctypes.c_void_p,
|
ctypes.c_void_p,
|
||||||
)
|
)
|
||||||
|
|
||||||
# Resolved lazily on first use.
|
# Resolved lazily on first use: (entry point, numAttrs to pass).
|
||||||
_batch_memcpy_fn: Any = None
|
_batch_memcpy: tuple[Any, int] | None = None
|
||||||
|
|
||||||
|
# Max copy descriptors per batch call, resolved lazily on first use.
|
||||||
|
_max_batch_descriptors: int | None = None
|
||||||
|
|
||||||
|
# ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so we chunk
|
||||||
|
# at that ceiling. Override via VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS.
|
||||||
|
_ROCM_DEFAULT_MAX_BATCH_DESCRIPTORS = 8192
|
||||||
|
|
||||||
|
|
||||||
def _resolve_batch_memcpy():
|
def _resolve_max_batch_descriptors() -> int:
|
||||||
"""Resolve the platform batch-memcpy entry point (one-time).
|
"""Max copy descriptors to pass to one batch-memcpy call (0 = unlimited).
|
||||||
|
|
||||||
* CUDA: ``cuMemcpyBatchAsync`` via ``cuGetProcAddress`` (uses
|
ROCm's ``hipMemcpyBatchAsync`` faults above 8192 descriptors per call, so
|
||||||
srcAccessOrder=STREAM via one attributes entry).
|
on ROCm we cap and chunk larger transfers. CUDA's ``cuMemcpyBatchAsync``
|
||||||
* ROCm: ``hipMemcpyBatchAsync`` from libamdhip64 (ROCm 7.1+). ROCm
|
handles arbitrary counts and is left uncapped. Set
|
||||||
7.2.1 or 7.2.2 rejects any call with ``numAttrs > 0``
|
``VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS`` (>0) to override on any platform.
|
||||||
(see ROCm/clr @ rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so
|
"""
|
||||||
we call with ``numAttrs=0``.
|
global _max_batch_descriptors
|
||||||
|
if _max_batch_descriptors is None:
|
||||||
|
override = envs.VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS
|
||||||
|
if override > 0:
|
||||||
|
_max_batch_descriptors = override
|
||||||
|
else:
|
||||||
|
_max_batch_descriptors = (
|
||||||
|
_ROCM_DEFAULT_MAX_BATCH_DESCRIPTORS if current_platform.is_rocm() else 0
|
||||||
|
)
|
||||||
|
return _max_batch_descriptors
|
||||||
|
|
||||||
Raises ``RuntimeError`` if the symbol is unavailable (older CUDA
|
|
||||||
driver, ROCm < 7.1, unusual install). The connector requires the
|
def _resolve_batch_memcpy() -> tuple[Any, int]:
|
||||||
batch API.
|
"""Resolve the batch-memcpy entry point and its ``numAttrs`` (one-time).
|
||||||
|
|
||||||
|
CUDA uses ``cuMemcpyBatchAsync``; ROCm uses ``hipMemcpyBatchAsync``.
|
||||||
|
Raises ``RuntimeError`` if the symbol is unavailable (old CUDA driver,
|
||||||
|
ROCm < 7.1, unusual install).
|
||||||
"""
|
"""
|
||||||
if current_platform.is_rocm():
|
if current_platform.is_rocm():
|
||||||
try:
|
try:
|
||||||
lib = ctypes.CDLL("libamdhip64.so", mode=ctypes.RTLD_GLOBAL)
|
lib = _load_hip_runtime()
|
||||||
fn = lib.hipMemcpyBatchAsync
|
fn = lib.hipMemcpyBatchAsync
|
||||||
except (OSError, AttributeError) as e:
|
except (OSError, AttributeError) as e:
|
||||||
raise RuntimeError(
|
raise RuntimeError(
|
||||||
@@ -97,14 +118,55 @@ def _resolve_batch_memcpy():
|
|||||||
ctypes.c_void_p, # failIdx
|
ctypes.c_void_p, # failIdx
|
||||||
ctypes.c_void_p, # stream
|
ctypes.c_void_p, # stream
|
||||||
]
|
]
|
||||||
return fn
|
return fn, _rocm_num_attrs(lib)
|
||||||
|
|
||||||
from cuda.bindings import driver as drv
|
from cuda.bindings import driver as drv
|
||||||
|
|
||||||
err, ptr, _ = drv.cuGetProcAddress(b"cuMemcpyBatchAsync", 12080, 0)
|
err, ptr, _ = drv.cuGetProcAddress(b"cuMemcpyBatchAsync", 12080, 0)
|
||||||
if err != drv.CUresult.CUDA_SUCCESS:
|
if err != drv.CUresult.CUDA_SUCCESS:
|
||||||
raise RuntimeError(f"cuGetProcAddress(cuMemcpyBatchAsync) failed: {err}")
|
raise RuntimeError(f"cuGetProcAddress(cuMemcpyBatchAsync) failed: {err}")
|
||||||
return _BATCH_MEMCPY_FUNC_TYPE(ptr)
|
return _BATCH_MEMCPY_FUNC_TYPE(ptr), 1
|
||||||
|
|
||||||
|
|
||||||
|
def _load_hip_runtime() -> ctypes.CDLL:
|
||||||
|
"""Load ``libamdhip64``, tolerating installs without the devel symlink.
|
||||||
|
|
||||||
|
The unversioned ``libamdhip64.so`` only ships with the ROCm devel package;
|
||||||
|
runtime-only and wheel-packaged ROCm installs provide just the versioned
|
||||||
|
soname. ``dlopen`` returns the already-mapped library when asked for a
|
||||||
|
soname the process has loaded — torch loads HIP at import — so the
|
||||||
|
versioned names resolve even when they are not on the loader search path.
|
||||||
|
"""
|
||||||
|
errors = []
|
||||||
|
for name in ("libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.6"):
|
||||||
|
try:
|
||||||
|
return ctypes.CDLL(name, mode=ctypes.RTLD_GLOBAL)
|
||||||
|
except OSError as e:
|
||||||
|
errors.append(f"{name}: {e}")
|
||||||
|
raise OSError("could not load the HIP runtime: " + "; ".join(errors))
|
||||||
|
|
||||||
|
|
||||||
|
def _rocm_num_attrs(lib: ctypes.CDLL) -> int:
|
||||||
|
"""``numAttrs`` for ``hipMemcpyBatchAsync`` on the running HIP runtime."""
|
||||||
|
ver = ctypes.c_int(0)
|
||||||
|
try:
|
||||||
|
if lib.hipRuntimeGetVersion(ctypes.byref(ver)) != 0:
|
||||||
|
ver.value = 0
|
||||||
|
except (OSError, AttributeError):
|
||||||
|
ver.value = 0
|
||||||
|
return _num_attrs_for_hip_version(ver.value)
|
||||||
|
|
||||||
|
|
||||||
|
def _num_attrs_for_hip_version(version: int) -> int:
|
||||||
|
"""``numAttrs`` for ``hipMemcpyBatchAsync`` given a HIP runtime version int.
|
||||||
|
|
||||||
|
ROCm 7.2.1-7.2.3 reject ``numAttrs > 0`` (ROCm/clr @ rocm-7.2.1
|
||||||
|
hipamd/src/hip_memory.cpp:2819-2822); 7.13+ accept it. ``version`` 0
|
||||||
|
(unknown) yields the conservative 0.
|
||||||
|
"""
|
||||||
|
# HIP encodes version as major*10_000_000 + minor*100_000 + patch.
|
||||||
|
major, minor = version // 10_000_000, (version // 100_000) % 100
|
||||||
|
return 1 if (major, minor) >= (7, 13) else 0
|
||||||
|
|
||||||
|
|
||||||
class BatchMemcpyParams(NamedTuple):
|
class BatchMemcpyParams(NamedTuple):
|
||||||
@@ -112,10 +174,12 @@ class BatchMemcpyParams(NamedTuple):
|
|||||||
dst_bases: np.ndarray # [num_layers] uint64
|
dst_bases: np.ndarray # [num_layers] uint64
|
||||||
bpb: np.ndarray # [num_layers] uint64 — bytes per block
|
bpb: np.ndarray # [num_layers] uint64 — bytes per block
|
||||||
num_layers: int
|
num_layers: int
|
||||||
# CUDA only: one attributes entry carrying srcAccessOrder. Unused on ROCm
|
# One attributes entry carrying srcAccessOrder. Ignored when num_attrs is
|
||||||
# (7.2.1 or 7.2.2) because the current runtime rejects numAttrs > 0.
|
# 0, which is what ROCm runtimes older than 7.13 require (see
|
||||||
|
# _num_attrs_for_hip_version).
|
||||||
attrs: _CUmemcpyAttributes
|
attrs: _CUmemcpyAttributes
|
||||||
attrs_idx: ctypes.c_size_t
|
attrs_idx: ctypes.c_size_t
|
||||||
|
num_attrs: int
|
||||||
# NOTE: cuMemcpyBatchAsync_v2() removed fail_idx field, but we use
|
# NOTE: cuMemcpyBatchAsync_v2() removed fail_idx field, but we use
|
||||||
# cuMemcpyBatchAsync() with fail_idx for backward compatibility
|
# cuMemcpyBatchAsync() with fail_idx for backward compatibility
|
||||||
fail_idx: ctypes.c_size_t
|
fail_idx: ctypes.c_size_t
|
||||||
@@ -128,9 +192,10 @@ def build_params(
|
|||||||
stream: torch.cuda.Stream,
|
stream: torch.cuda.Stream,
|
||||||
src_access_order: int = CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
|
src_access_order: int = CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
|
||||||
) -> BatchMemcpyParams:
|
) -> BatchMemcpyParams:
|
||||||
global _batch_memcpy_fn
|
global _batch_memcpy
|
||||||
if _batch_memcpy_fn is None:
|
if _batch_memcpy is None:
|
||||||
_batch_memcpy_fn = _resolve_batch_memcpy()
|
_batch_memcpy = _resolve_batch_memcpy()
|
||||||
|
_, num_attrs = _batch_memcpy
|
||||||
|
|
||||||
assert list(src_caches.keys()) == list(dst_caches.keys())
|
assert list(src_caches.keys()) == list(dst_caches.keys())
|
||||||
src_tensors = list(src_caches.values())
|
src_tensors = list(src_caches.values())
|
||||||
@@ -153,6 +218,7 @@ def build_params(
|
|||||||
num_layers=len(src_tensors),
|
num_layers=len(src_tensors),
|
||||||
attrs=attrs,
|
attrs=attrs,
|
||||||
attrs_idx=ctypes.c_size_t(0),
|
attrs_idx=ctypes.c_size_t(0),
|
||||||
|
num_attrs=num_attrs,
|
||||||
fail_idx=ctypes.c_size_t(0),
|
fail_idx=ctypes.c_size_t(0),
|
||||||
stream_handle=stream.cuda_stream,
|
stream_handle=stream.cuda_stream,
|
||||||
)
|
)
|
||||||
@@ -168,6 +234,9 @@ def copy_blocks(
|
|||||||
if n == 0:
|
if n == 0:
|
||||||
return
|
return
|
||||||
|
|
||||||
|
assert _batch_memcpy is not None, "build_params() must run before copy_blocks()"
|
||||||
|
fn, _ = _batch_memcpy
|
||||||
|
|
||||||
src_ids = np.array(src_block_ids, dtype=np.uint64)
|
src_ids = np.array(src_block_ids, dtype=np.uint64)
|
||||||
dst_ids = np.array(dst_block_ids, dtype=np.uint64)
|
dst_ids = np.array(dst_block_ids, dtype=np.uint64)
|
||||||
|
|
||||||
@@ -180,23 +249,24 @@ def copy_blocks(
|
|||||||
sz_all = np.repeat(params.bpb, n)
|
sz_all = np.repeat(params.bpb, n)
|
||||||
total = n * params.num_layers
|
total = n * params.num_layers
|
||||||
|
|
||||||
# ROCm 7.2.1/7.2.2 rejects any call with numAttrs>0 (hipMemcpyBatchAsync
|
# Chunk on ROCm: hipMemcpyBatchAsync faults above 8192 descriptors/call.
|
||||||
# hipamd/src/hip_memory.cpp:2819-2822); CUDA uses one attrs entry so
|
# CUDA is uncapped (max_desc == 0) and issues a single call.
|
||||||
# srcAccessOrder is honored. attrs / attrsIdxs are ignored when
|
max_desc = _resolve_max_batch_descriptors()
|
||||||
# numAttrs==0, so we pass the same values from both paths.
|
step = total if max_desc <= 0 else max_desc
|
||||||
num_attrs = 0 if current_platform.is_rocm() else 1
|
for off in range(0, total, step):
|
||||||
err = _batch_memcpy_fn(
|
cnt = min(step, total - off)
|
||||||
dst_all.ctypes.data,
|
err = fn(
|
||||||
src_all.ctypes.data,
|
dst_all[off : off + cnt].ctypes.data,
|
||||||
sz_all.ctypes.data,
|
src_all[off : off + cnt].ctypes.data,
|
||||||
total,
|
sz_all[off : off + cnt].ctypes.data,
|
||||||
ctypes.addressof(params.attrs),
|
cnt,
|
||||||
ctypes.byref(params.attrs_idx),
|
ctypes.addressof(params.attrs),
|
||||||
num_attrs,
|
ctypes.byref(params.attrs_idx),
|
||||||
ctypes.byref(params.fail_idx),
|
params.num_attrs,
|
||||||
params.stream_handle,
|
ctypes.byref(params.fail_idx),
|
||||||
)
|
params.stream_handle,
|
||||||
if err != 0:
|
|
||||||
raise RuntimeError(
|
|
||||||
f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}"
|
|
||||||
)
|
)
|
||||||
|
if err != 0:
|
||||||
|
raise RuntimeError(
|
||||||
|
f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}"
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user