[Bugfix][KV Offload] Fix swap_blocks_batch on the default stream (#46379)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <Itay.etelis@gmail.com>
This commit is contained in:
Itay Etelis
2026-06-23 05:45:27 -07:00
committed by GitHub
co-authored by Itay Etelis Itay Etelis
parent 091bc1026e
commit 7d47cff933
2 changed files with 48 additions and 1 deletions
+6 -1
View File
@@ -127,7 +127,12 @@ void swap_blocks_batch(const torch::stable::Tensor& src_ptrs,
return reinterpret_cast<BatchFn>(fn_ptr);
}();
if (batch_fn != nullptr) {
// cuMemcpyBatchAsync rejects the legacy default stream (handle 0 /
// cudaStreamLegacy) with CUDA_ERROR_INVALID_VALUE; route it to the per-copy
// fallback below, which is correct on any stream. Real and per-thread-default
// streams take the batch fast path.
const bool usable_stream = stream != nullptr && stream != cudaStreamLegacy;
if (batch_fn != nullptr && usable_stream) {
CUmemcpyAttributes attr = {};
// ANY lets the DMA engine prefetch source bytes out of stream order,
// which is only safe when no GPU stream is concurrently writing the
@@ -0,0 +1,42 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit tests for the ``ops.swap_blocks_batch`` C++ (cuMemcpyBatchAsync) path."""
import pytest
import torch
from vllm import _custom_ops as ops
from vllm.platforms import current_platform
def _addrs(buffers: list[torch.Tensor]) -> torch.Tensor:
return torch.tensor([b.data_ptr() for b in buffers], dtype=torch.int64)
def _run_batch(sizes: list[int]) -> None:
src = [torch.randint(256, (s,), dtype=torch.uint8, device="cuda") for s in sizes]
dst = [torch.zeros_like(s) for s in src]
ops.swap_blocks_batch(
_addrs(src), _addrs(dst), torch.tensor(sizes, dtype=torch.int64)
)
torch.accelerator.synchronize()
for s, d in zip(src, dst):
assert torch.equal(d, s)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="swap_blocks_batch requires CUDA"
)
def test_swap_blocks_batch_default_stream():
# cuMemcpyBatchAsync rejects the legacy default stream; the op must fall
# back to per-copy transfers instead of raising.
_run_batch([8, 4096, 8192])
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="swap_blocks_batch requires CUDA"
)
def test_swap_blocks_batch_dedicated_stream():
# A dedicated non-default stream exercises the cuMemcpyBatchAsync fast path.
with torch.cuda.stream(torch.cuda.Stream()):
_run_batch([8, 4096, 8192])