diff --git a/csrc/libtorch_stable/cache_kernels.cu b/csrc/libtorch_stable/cache_kernels.cu index ebeba380af..a1ac81cb10 100644 --- a/csrc/libtorch_stable/cache_kernels.cu +++ b/csrc/libtorch_stable/cache_kernels.cu @@ -127,7 +127,12 @@ void swap_blocks_batch(const torch::stable::Tensor& src_ptrs, return reinterpret_cast(fn_ptr); }(); - if (batch_fn != nullptr) { + // cuMemcpyBatchAsync rejects the legacy default stream (handle 0 / + // cudaStreamLegacy) with CUDA_ERROR_INVALID_VALUE; route it to the per-copy + // fallback below, which is correct on any stream. Real and per-thread-default + // streams take the batch fast path. + const bool usable_stream = stream != nullptr && stream != cudaStreamLegacy; + if (batch_fn != nullptr && usable_stream) { CUmemcpyAttributes attr = {}; // ANY lets the DMA engine prefetch source bytes out of stream order, // which is only safe when no GPU stream is concurrently writing the diff --git a/tests/v1/kv_offload/cpu/test_swap_blocks_batch.py b/tests/v1/kv_offload/cpu/test_swap_blocks_batch.py new file mode 100644 index 0000000000..cc89759ba1 --- /dev/null +++ b/tests/v1/kv_offload/cpu/test_swap_blocks_batch.py @@ -0,0 +1,42 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Unit tests for the ``ops.swap_blocks_batch`` C++ (cuMemcpyBatchAsync) path.""" + +import pytest +import torch + +from vllm import _custom_ops as ops +from vllm.platforms import current_platform + + +def _addrs(buffers: list[torch.Tensor]) -> torch.Tensor: + return torch.tensor([b.data_ptr() for b in buffers], dtype=torch.int64) + + +def _run_batch(sizes: list[int]) -> None: + src = [torch.randint(256, (s,), dtype=torch.uint8, device="cuda") for s in sizes] + dst = [torch.zeros_like(s) for s in src] + ops.swap_blocks_batch( + _addrs(src), _addrs(dst), torch.tensor(sizes, dtype=torch.int64) + ) + torch.accelerator.synchronize() + for s, d in zip(src, dst): + assert torch.equal(d, s) + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="swap_blocks_batch requires CUDA" +) +def test_swap_blocks_batch_default_stream(): + # cuMemcpyBatchAsync rejects the legacy default stream; the op must fall + # back to per-copy transfers instead of raising. + _run_batch([8, 4096, 8192]) + + +@pytest.mark.skipif( + not current_platform.is_cuda(), reason="swap_blocks_batch requires CUDA" +) +def test_swap_blocks_batch_dedicated_stream(): + # A dedicated non-default stream exercises the cuMemcpyBatchAsync fast path. + with torch.cuda.stream(torch.cuda.Stream()): + _run_batch([8, 4096, 8192])