diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index 2dbd498fb3..6b8908d2cd 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -16,6 +16,7 @@ #include #include #include +#include #ifdef USE_ROCM #include @@ -76,6 +77,31 @@ void swap_blocks(torch::stable::Tensor& src, torch::stable::Tensor& dst, } } +namespace { +// ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so chunk +// at that ceiling. +constexpr int64_t kRocmDefaultMaxBatchDescriptors = 8192; + +constexpr const char* kMaxBatchDescriptorsEnv = + "VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS"; + +// Max copy descriptors per batch-memcpy call (0 = unlimited). ROCm caps and +// chunks; CUDA is uncapped. The env var (>0) overrides on any platform. +int64_t resolve_max_batch_descriptors() { + static const int64_t cached = []() -> int64_t { + const char* val = std::getenv(kMaxBatchDescriptorsEnv); + const int64_t override_val = val ? std::atoll(val) : 0; + if (override_val > 0) return override_val; +#if defined(USE_ROCM) + return kRocmDefaultMaxBatchDescriptors; +#else + return 0; +#endif + }(); + return cached; +} +} // namespace + void swap_blocks_batch(const torch::stable::Tensor& src_ptrs, const torch::stable::Tensor& dst_ptrs, const torch::stable::Tensor& sizes, @@ -142,32 +168,63 @@ void swap_blocks_batch(const torch::stable::Tensor& src_ptrs, : CU_MEMCPY_SRC_ACCESS_ORDER_STREAM; size_t attrs_idx = 0; size_t fail_idx = 0; - CUresult result = batch_fn(reinterpret_cast(dst_data), - reinterpret_cast(src_data), - reinterpret_cast(size_data), - static_cast(n), &attr, &attrs_idx, 1, - &fail_idx, static_cast(stream)); - STD_TORCH_CHECK(result == CUDA_SUCCESS, - "cuMemcpyBatchAsync failed at index ", fail_idx, - " with error ", result); + // Uncapped on CUDA (max_desc == 0 -> single call) unless overridden by + // VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS; chunk to honor the override. + const int64_t max_desc = resolve_max_batch_descriptors(); + const int64_t step = max_desc <= 0 ? n : max_desc; + for (int64_t off = 0; off < n; off += step) { + const int64_t cnt = std::min(step, n - off); + CUresult result = batch_fn(reinterpret_cast(dst_data + off), + reinterpret_cast(src_data + off), + reinterpret_cast(size_data + off), + static_cast(cnt), &attr, &attrs_idx, 1, + &fail_idx, static_cast(stream)); + STD_TORCH_CHECK(result == CUDA_SUCCESS, + "cuMemcpyBatchAsync failed at index ", fail_idx, + " with error ", result); + } return; } #elif defined(USE_ROCM) && defined(HIP_VERSION) && HIP_VERSION >= 70100000 - // ROCm 7.1+ exposes hipMemcpyBatchAsync. The 7.2.1 implementation early- - // returns hipErrorNotSupported whenever numAttrs > 0 (see ROCm/clr @ - // rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so call with - // numAttrs=0. + // ROCm 7.1+ exposes hipMemcpyBatchAsync. ROCm 7.2.1-7.2.3 early-return + // hipErrorNotSupported whenever numAttrs > 0 (see ROCm/clr @ rocm-7.2.1 + // hipamd/src/hip_memory.cpp:2819-2822), so those releases must call with + // numAttrs=0. ROCm 7.13+ accepts numAttrs > 0. + // rocm-7.14+ has better performance. { hipMemcpyAttributes attr = {}; size_t attrs_idx = 0; size_t fail_idx = 0; - hipError_t result = hipMemcpyBatchAsync( - reinterpret_cast(dst_data), reinterpret_cast(src_data), - reinterpret_cast(size_data), static_cast(n), &attr, - &attrs_idx, 0, &fail_idx, static_cast(stream)); - STD_TORCH_CHECK(result == hipSuccess, - "hipMemcpyBatchAsync failed at index ", fail_idx, - " with error ", result); + size_t num_attrs = 0; + #if HIP_VERSION >= 71300000 + static const bool runtime_accepts_attrs = []() { + int runtime_version = 0; + return hipRuntimeGetVersion(&runtime_version) == hipSuccess && + runtime_version >= 71300000; + }(); + if (runtime_accepts_attrs) { + attr.srcAccessOrder = is_src_access_order_any + ? hipMemcpySrcAccessOrderAny + : hipMemcpySrcAccessOrderStream; + num_attrs = 1; + } + #endif + // hipMemcpyBatchAsync faults (GPU memory access fault) above 8192 + // descriptors/call on ROCm 7.15, so chunk the batch at the resolved cap. + const int64_t max_desc = resolve_max_batch_descriptors(); + const int64_t step = max_desc <= 0 ? n : max_desc; + for (int64_t off = 0; off < n; off += step) { + const int64_t cnt = std::min(step, n - off); + hipError_t result = hipMemcpyBatchAsync( + reinterpret_cast(dst_data + off), + reinterpret_cast(src_data + off), + reinterpret_cast(size_data + off), static_cast(cnt), + &attr, &attrs_idx, num_attrs, &fail_idx, + static_cast(stream)); + STD_TORCH_CHECK(result == hipSuccess, + "hipMemcpyBatchAsync failed at index ", fail_idx, + " with error ", result); + } return; } #endif diff --git a/tests/v1/simple_kv_offload/test_hip_mem_ops.py b/tests/v1/simple_kv_offload/test_hip_mem_ops.py new file mode 100644 index 0000000000..c30d2844ce --- /dev/null +++ b/tests/v1/simple_kv_offload/test_hip_mem_ops.py @@ -0,0 +1,145 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the HIP/ROCm paths in vllm.v1.simple_kv_offload.cuda_mem_ops. + +Deliberately not skipped off ROCm: the version gate takes a plain version int +and the library loader is monkeypatched, so nothing here needs a HIP runtime +and the logic stays covered by the default (NVIDIA) CI that runs this +directory. +""" + +import ctypes + +import numpy as np +import pytest + +from vllm.v1.simple_kv_offload import cuda_mem_ops +from vllm.v1.simple_kv_offload.cuda_mem_ops import ( + BatchMemcpyParams, + _CUmemcpyAttributes, + _load_hip_runtime, + _num_attrs_for_hip_version, + copy_blocks, +) + + +# hipMemcpyBatchAsync rejects numAttrs > 0 on ROCm 7.2.1-7.2.3 and accepts it +# on 7.13+. HIP encodes the version as major*10_000_000 + minor*100_000 + patch. +@pytest.mark.parametrize( + ("version", "expected"), + [ + (0, 0), # unknown (hipRuntimeGetVersion failed) -> conservative 0 + (70_100_000, 0), # ROCm 7.1 + (70_200_000, 0), # ROCm 7.2.0 + (70_253_211, 0), # ROCm 7.2.2 (patch 53211) + (70_300_000, 0), # ROCm 7.3 + (71_200_000, 0), # ROCm 7.12 -- just below the 7.13 cutoff + (71_300_000, 1), # ROCm 7.13 -- first release accepting numAttrs > 0 + (71_326_193, 1), # ROCm 7.13 nightly (patch 26193) + (71_400_000, 1), # ROCm 7.14 + (71_500_000, 1), # ROCm 7.15 + (80_000_000, 1), # ROCm 8.0 + ], +) +def test_num_attrs_for_hip_version(version: int, expected: int): + assert _num_attrs_for_hip_version(version) == expected + + +@pytest.mark.parametrize( + ("available", "expected"), + [ + # Devel install: the unversioned symlink is present and preferred. + ({"libamdhip64.so", "libamdhip64.so.7"}, "libamdhip64.so"), + # Runtime-only / wheel-packaged ROCm: versioned soname only. + ({"libamdhip64.so.7"}, "libamdhip64.so.7"), + ({"libamdhip64.so.6"}, "libamdhip64.so.6"), + ], +) +def test_load_hip_runtime_falls_back_to_versioned_soname( + monkeypatch, available: set[str], expected: str +): + loaded = [] + + def fake_cdll(name, mode=0): + if name not in available: + raise OSError(f"{name}: cannot open shared object file") + loaded.append(name) + return name + + monkeypatch.setattr(cuda_mem_ops.ctypes, "CDLL", fake_cdll) + assert _load_hip_runtime() == expected + assert loaded == [expected] + + +def test_load_hip_runtime_reports_every_candidate_when_all_fail(monkeypatch): + def fake_cdll(name, mode=0): + raise OSError("cannot open shared object file") + + monkeypatch.setattr(cuda_mem_ops.ctypes, "CDLL", fake_cdll) + with pytest.raises(OSError, match="could not load the HIP runtime") as exc: + _load_hip_runtime() + for name in ("libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.6"): + assert name in str(exc.value) + + +def _fake_params(num_layers: int, bytes_per_block: int = 512) -> BatchMemcpyParams: + """Minimal params for driving copy_blocks() without a real device. + + Bases are 0 and the batch-memcpy function is monkeypatched, so no memory is + ever dereferenced; only the descriptor bookkeeping (count/chunking) matters. + """ + return BatchMemcpyParams( + src_bases=np.zeros(num_layers, dtype=np.uint64), + dst_bases=np.zeros(num_layers, dtype=np.uint64), + bpb=np.full(num_layers, bytes_per_block, dtype=np.uint64), + num_layers=num_layers, + attrs=_CUmemcpyAttributes(), + attrs_idx=ctypes.c_size_t(0), + num_attrs=0, + fail_idx=ctypes.c_size_t(0), + stream_handle=0, + ) + + +def _record_counts(monkeypatch) -> list[int]: + """Patch the batch-memcpy call to record the descriptor count per call.""" + counts: list[int] = [] + + def fake_fn(dst, src, sizes, count, *rest): + counts.append(count) + return 0 + + monkeypatch.setattr(cuda_mem_ops, "_batch_memcpy", (fake_fn, 0)) + return counts + + +# ROCm chunks the descriptor batch at the cap (hipMemcpyBatchAsync faults above +# 8192/call); CUDA leaves the cap at 0 and issues a single call. +@pytest.mark.parametrize( + ("max_desc", "num_layers", "num_blocks", "expected_counts"), + [ + (8192, 3, 5000, [8192, 6808]), # 15000 desc -> 8192 + 6808 + (8192, 1, 8192, [8192]), # exactly at the cap -> single call + (8192, 1, 8193, [8192, 1]), # one over -> spills into a second call + (4096, 61, 135, [4096, 4096, 43]), # bs=1 Kimi shape (8235 desc) + (0, 61, 135, [8235]), # CUDA uncapped -> single call + ], +) +def test_copy_blocks_chunks_at_descriptor_cap( + monkeypatch, max_desc, num_layers, num_blocks, expected_counts +): + counts = _record_counts(monkeypatch) + monkeypatch.setattr(cuda_mem_ops, "_max_batch_descriptors", max_desc) + + ids = list(range(num_blocks)) + copy_blocks(ids, ids, _fake_params(num_layers)) + + assert counts == expected_counts + assert sum(counts) == num_layers * num_blocks + + +def test_copy_blocks_noop_on_empty(monkeypatch): + counts = _record_counts(monkeypatch) + monkeypatch.setattr(cuda_mem_ops, "_max_batch_descriptors", 8192) + copy_blocks([], [], _fake_params(num_layers=4)) + assert counts == [] diff --git a/vllm/envs.py b/vllm/envs.py index 5973b58507..2eba6649c5 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -299,6 +299,7 @@ if TYPE_CHECKING: VLLM_DEBUG_MFU_METRICS: bool = False VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False VLLM_WEIGHT_OFFLOADING_DISABLE_UVA: bool = False + VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS: int = 0 VLLM_WSL2_ENABLE_PIN_MEMORY: bool = False VLLM_DISABLE_LOG_LOGO: bool = False VLLM_LORA_DISABLE_PDL: bool = False @@ -2059,6 +2060,12 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_WEIGHT_OFFLOADING_DISABLE_UVA": lambda: bool( int(os.getenv("VLLM_WEIGHT_OFFLOADING_DISABLE_UVA", "0")) ), + # Max descriptors per CPU-KV-offload batch-memcpy call. 0 = platform default + # (ROCm chunks at 8192, since hipMemcpyBatchAsync faults above that on + # rocm 7.14/7.15 when batch copy is optimized; CUDA uncapped). Set >0 to override. + "VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS": lambda: int( + os.getenv("VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS", "0") + ), # On WSL2 with a compatible kernel (>= 4.19.121), pinned memory is # supported but disabled by default due to a small performance regression. # Set to 1 when pinned memory or UVA is required (e.g. CPU offloading diff --git a/vllm/v1/simple_kv_offload/cuda_mem_ops.py b/vllm/v1/simple_kv_offload/cuda_mem_ops.py index 69b1677e0a..1f3044155e 100644 --- a/vllm/v1/simple_kv_offload/cuda_mem_ops.py +++ b/vllm/v1/simple_kv_offload/cuda_mem_ops.py @@ -8,6 +8,7 @@ from typing import Any, NamedTuple import numpy as np import torch +import vllm.envs as envs from vllm.logger import init_logger from vllm.platforms import current_platform @@ -58,27 +59,47 @@ _BATCH_MEMCPY_FUNC_TYPE = ctypes.CFUNCTYPE( ctypes.c_void_p, ) -# Resolved lazily on first use. -_batch_memcpy_fn: Any = None +# Resolved lazily on first use: (entry point, numAttrs to pass). +_batch_memcpy: tuple[Any, int] | None = None + +# Max copy descriptors per batch call, resolved lazily on first use. +_max_batch_descriptors: int | None = None + +# ROCm hipMemcpyBatchAsync faults for count > 8192 (MI350X/gfx950), so we chunk +# at that ceiling. Override via VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS. +_ROCM_DEFAULT_MAX_BATCH_DESCRIPTORS = 8192 -def _resolve_batch_memcpy(): - """Resolve the platform batch-memcpy entry point (one-time). +def _resolve_max_batch_descriptors() -> int: + """Max copy descriptors to pass to one batch-memcpy call (0 = unlimited). - * CUDA: ``cuMemcpyBatchAsync`` via ``cuGetProcAddress`` (uses - srcAccessOrder=STREAM via one attributes entry). - * ROCm: ``hipMemcpyBatchAsync`` from libamdhip64 (ROCm 7.1+). ROCm - 7.2.1 or 7.2.2 rejects any call with ``numAttrs > 0`` - (see ROCm/clr @ rocm-7.2.1 hipamd/src/hip_memory.cpp:2819-2822), so - we call with ``numAttrs=0``. + ROCm's ``hipMemcpyBatchAsync`` faults above 8192 descriptors per call, so + on ROCm we cap and chunk larger transfers. CUDA's ``cuMemcpyBatchAsync`` + handles arbitrary counts and is left uncapped. Set + ``VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS`` (>0) to override on any platform. + """ + global _max_batch_descriptors + if _max_batch_descriptors is None: + override = envs.VLLM_KV_OFFLOAD_MAX_BATCH_DESCRIPTORS + if override > 0: + _max_batch_descriptors = override + else: + _max_batch_descriptors = ( + _ROCM_DEFAULT_MAX_BATCH_DESCRIPTORS if current_platform.is_rocm() else 0 + ) + return _max_batch_descriptors - Raises ``RuntimeError`` if the symbol is unavailable (older CUDA - driver, ROCm < 7.1, unusual install). The connector requires the - batch API. + +def _resolve_batch_memcpy() -> tuple[Any, int]: + """Resolve the batch-memcpy entry point and its ``numAttrs`` (one-time). + + CUDA uses ``cuMemcpyBatchAsync``; ROCm uses ``hipMemcpyBatchAsync``. + Raises ``RuntimeError`` if the symbol is unavailable (old CUDA driver, + ROCm < 7.1, unusual install). """ if current_platform.is_rocm(): try: - lib = ctypes.CDLL("libamdhip64.so", mode=ctypes.RTLD_GLOBAL) + lib = _load_hip_runtime() fn = lib.hipMemcpyBatchAsync except (OSError, AttributeError) as e: raise RuntimeError( @@ -97,14 +118,55 @@ def _resolve_batch_memcpy(): ctypes.c_void_p, # failIdx ctypes.c_void_p, # stream ] - return fn + return fn, _rocm_num_attrs(lib) from cuda.bindings import driver as drv err, ptr, _ = drv.cuGetProcAddress(b"cuMemcpyBatchAsync", 12080, 0) if err != drv.CUresult.CUDA_SUCCESS: raise RuntimeError(f"cuGetProcAddress(cuMemcpyBatchAsync) failed: {err}") - return _BATCH_MEMCPY_FUNC_TYPE(ptr) + return _BATCH_MEMCPY_FUNC_TYPE(ptr), 1 + + +def _load_hip_runtime() -> ctypes.CDLL: + """Load ``libamdhip64``, tolerating installs without the devel symlink. + + The unversioned ``libamdhip64.so`` only ships with the ROCm devel package; + runtime-only and wheel-packaged ROCm installs provide just the versioned + soname. ``dlopen`` returns the already-mapped library when asked for a + soname the process has loaded — torch loads HIP at import — so the + versioned names resolve even when they are not on the loader search path. + """ + errors = [] + for name in ("libamdhip64.so", "libamdhip64.so.7", "libamdhip64.so.6"): + try: + return ctypes.CDLL(name, mode=ctypes.RTLD_GLOBAL) + except OSError as e: + errors.append(f"{name}: {e}") + raise OSError("could not load the HIP runtime: " + "; ".join(errors)) + + +def _rocm_num_attrs(lib: ctypes.CDLL) -> int: + """``numAttrs`` for ``hipMemcpyBatchAsync`` on the running HIP runtime.""" + ver = ctypes.c_int(0) + try: + if lib.hipRuntimeGetVersion(ctypes.byref(ver)) != 0: + ver.value = 0 + except (OSError, AttributeError): + ver.value = 0 + return _num_attrs_for_hip_version(ver.value) + + +def _num_attrs_for_hip_version(version: int) -> int: + """``numAttrs`` for ``hipMemcpyBatchAsync`` given a HIP runtime version int. + + ROCm 7.2.1-7.2.3 reject ``numAttrs > 0`` (ROCm/clr @ rocm-7.2.1 + hipamd/src/hip_memory.cpp:2819-2822); 7.13+ accept it. ``version`` 0 + (unknown) yields the conservative 0. + """ + # HIP encodes version as major*10_000_000 + minor*100_000 + patch. + major, minor = version // 10_000_000, (version // 100_000) % 100 + return 1 if (major, minor) >= (7, 13) else 0 class BatchMemcpyParams(NamedTuple): @@ -112,10 +174,12 @@ class BatchMemcpyParams(NamedTuple): dst_bases: np.ndarray # [num_layers] uint64 bpb: np.ndarray # [num_layers] uint64 — bytes per block num_layers: int - # CUDA only: one attributes entry carrying srcAccessOrder. Unused on ROCm - # (7.2.1 or 7.2.2) because the current runtime rejects numAttrs > 0. + # One attributes entry carrying srcAccessOrder. Ignored when num_attrs is + # 0, which is what ROCm runtimes older than 7.13 require (see + # _num_attrs_for_hip_version). attrs: _CUmemcpyAttributes attrs_idx: ctypes.c_size_t + num_attrs: int # NOTE: cuMemcpyBatchAsync_v2() removed fail_idx field, but we use # cuMemcpyBatchAsync() with fail_idx for backward compatibility fail_idx: ctypes.c_size_t @@ -128,9 +192,10 @@ def build_params( stream: torch.cuda.Stream, src_access_order: int = CU_MEMCPY_SRC_ACCESS_ORDER_ANY, ) -> BatchMemcpyParams: - global _batch_memcpy_fn - if _batch_memcpy_fn is None: - _batch_memcpy_fn = _resolve_batch_memcpy() + global _batch_memcpy + if _batch_memcpy is None: + _batch_memcpy = _resolve_batch_memcpy() + _, num_attrs = _batch_memcpy assert list(src_caches.keys()) == list(dst_caches.keys()) src_tensors = list(src_caches.values()) @@ -153,6 +218,7 @@ def build_params( num_layers=len(src_tensors), attrs=attrs, attrs_idx=ctypes.c_size_t(0), + num_attrs=num_attrs, fail_idx=ctypes.c_size_t(0), stream_handle=stream.cuda_stream, ) @@ -168,6 +234,9 @@ def copy_blocks( if n == 0: return + assert _batch_memcpy is not None, "build_params() must run before copy_blocks()" + fn, _ = _batch_memcpy + src_ids = np.array(src_block_ids, dtype=np.uint64) dst_ids = np.array(dst_block_ids, dtype=np.uint64) @@ -180,23 +249,24 @@ def copy_blocks( sz_all = np.repeat(params.bpb, n) total = n * params.num_layers - # ROCm 7.2.1/7.2.2 rejects any call with numAttrs>0 (hipMemcpyBatchAsync - # hipamd/src/hip_memory.cpp:2819-2822); CUDA uses one attrs entry so - # srcAccessOrder is honored. attrs / attrsIdxs are ignored when - # numAttrs==0, so we pass the same values from both paths. - num_attrs = 0 if current_platform.is_rocm() else 1 - err = _batch_memcpy_fn( - dst_all.ctypes.data, - src_all.ctypes.data, - sz_all.ctypes.data, - total, - ctypes.addressof(params.attrs), - ctypes.byref(params.attrs_idx), - num_attrs, - ctypes.byref(params.fail_idx), - params.stream_handle, - ) - if err != 0: - raise RuntimeError( - f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}" + # Chunk on ROCm: hipMemcpyBatchAsync faults above 8192 descriptors/call. + # CUDA is uncapped (max_desc == 0) and issues a single call. + max_desc = _resolve_max_batch_descriptors() + step = total if max_desc <= 0 else max_desc + for off in range(0, total, step): + cnt = min(step, total - off) + err = fn( + dst_all[off : off + cnt].ctypes.data, + src_all[off : off + cnt].ctypes.data, + sz_all[off : off + cnt].ctypes.data, + cnt, + ctypes.addressof(params.attrs), + ctypes.byref(params.attrs_idx), + params.num_attrs, + ctypes.byref(params.fail_idx), + params.stream_handle, ) + if err != 0: + raise RuntimeError( + f"batch memcpy failed: err={err} failIdx={params.fail_idx.value}" + )