[Perf] Triton fast path for small CPU→GPU swap_blocks_batch in the offloading connector (#42212)

Signed-off-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: Itay Etelis <itay.etelis@ibm.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Itay Etelis
2026-06-03 13:38:17 +03:00
committed by GitHub
co-authored by Itay Etelis mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 02564b4de0
commit 1fa9ea09f6
3 changed files with 180 additions and 11 deletions
@@ -0,0 +1,31 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Unit test for the Triton ``swap_blocks_batch`` fast-path kernel."""
import pytest
import torch
from vllm.platforms import current_platform
from vllm.v1.kv_offload.cpu.swap_blocks_triton import swap_blocks_batch
def _addrs(buffers: list[torch.Tensor]) -> torch.Tensor:
return torch.tensor([b.data_ptr() for b in buffers], dtype=torch.int64)
@pytest.mark.skipif(
not current_platform.is_cuda(), reason="Triton swap fast path requires CUDA"
)
def test_triton_swap_copies_source_bytes():
# 8-byte-aligned, sub-threshold sizes covering 8 KiB chunk boundaries and
# odd tail-mask lengths, with enough descriptors to take the Triton path.
sizes = [8, 4096, 8192, 8200, 16384, 4088] * 8
src = [torch.randint(256, (s,), dtype=torch.uint8, device="cuda") for s in sizes]
dst = [torch.zeros_like(s) for s in src]
sizes_t = torch.tensor(sizes, dtype=torch.int64)
swap_blocks_batch(_addrs(src), _addrs(dst), sizes_t.clone(), bytes_per_chunk=8192)
torch.accelerator.synchronize()
for s, t in zip(src, dst):
assert torch.equal(t, s) # kernel copied the source bytes verbatim
+75 -11
View File
@@ -1,5 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import functools
import time
from collections import deque
from dataclasses import dataclass
@@ -10,6 +11,7 @@ from typing_extensions import override
from vllm import _custom_ops as ops
from vllm.logger import init_logger
from vllm.triton_utils import HAS_TRITON, triton
from vllm.utils.math_utils import cdiv
from vllm.utils.platform_utils import is_pin_memory_available
from vllm.v1.kv_offload.base import (
@@ -19,6 +21,10 @@ from vllm.v1.kv_offload.base import (
GPULoadStoreSpec,
)
from vllm.v1.kv_offload.cpu.shared_offload_region import SharedOffloadRegion
from vllm.v1.kv_offload.cpu.swap_blocks_triton import (
THRESHOLD_BYTES,
swap_blocks_batch,
)
from vllm.v1.kv_offload.worker.worker import (
OffloadingHandler,
TransferResult,
@@ -28,6 +34,30 @@ from vllm.v1.kv_offload.worker.worker import (
logger = init_logger(__name__)
def _select_swap_blocks_fn(
kv_cache_groups_data_refs: list[list[CanonicalKVCacheRef]],
gpu_to_cpu: bool,
):
"""Resolve the swap_blocks function for a handler at init time."""
# GPU->CPU is bandwidth-bound; the dedicated copy engine beats Triton.
if gpu_to_cpu:
return ops.swap_blocks_batch
# Fall back to the C++ DMA path on platforms where Triton isn't usable
# (e.g. ROCm builds without Triton).
if not HAS_TRITON:
return ops.swap_blocks_batch
page_sizes = [r.page_size_bytes for g in kv_cache_groups_data_refs for r in g]
# Triton wins only on small, 8-byte-aligned payloads.
if (
not page_sizes
or max(page_sizes) >= THRESHOLD_BYTES
or any(s % 8 for s in page_sizes)
):
return ops.swap_blocks_batch
chunk = min(triton.next_power_of_2(max(page_sizes)), 8192)
return functools.partial(swap_blocks_batch, bytes_per_chunk=chunk)
@dataclass
class Transfer:
job_id: int
@@ -35,6 +65,9 @@ class Transfer:
start_event: torch.Event
end_event: torch.Event
num_bytes: int
batch_src: torch.Tensor
batch_dst: torch.Tensor
batch_sizes: torch.Tensor
def compute_sub_block_ptrs(
@@ -109,6 +142,17 @@ def pin_mmap_region(region: SharedOffloadRegion) -> None:
region.is_pinned = True
def _new_descriptor_buffers(
num_copy_ops: int,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
pin = is_pin_memory_available()
return (
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
torch.empty(num_copy_ops, dtype=torch.int64, pin_memory=pin),
)
class SingleDirectionOffloadingHandler(OffloadingHandler):
"""
SingleDirectionOffloadingHandler handles transfers for a single direction,
@@ -162,6 +206,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
)
self.gpu_to_cpu: bool = gpu_to_cpu
self.kv_cache_groups_data_refs = kv_cache_groups_data_refs
self._swap_blocks_batch = _select_swap_blocks_fn(
kv_cache_groups_data_refs, gpu_to_cpu
)
# GPU blocks may be smaller
# cpu_page_size = gpu_page_size * block_size_factor.
@@ -179,6 +226,8 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self._stream_pool: list[torch.cuda.Stream] = []
# list of CUDA events available for re-use
self._event_pool: list[torch.Event] = []
# list of pinned descriptor buffer sets available for re-use
self._buffer_pool: list[tuple[torch.Tensor, torch.Tensor, torch.Tensor]] = []
@override
def transfer_async(self, job_id: int, transfer_spec: TransferSpec) -> bool:
@@ -229,9 +278,21 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
):
num_copy_ops += group_size * len(group_data_refs)
all_src = np.empty(num_copy_ops, dtype=np.int64)
all_dst = np.empty(num_copy_ops, dtype=np.int64)
all_sizes = np.empty(num_copy_ops, dtype=np.int64)
# reuse a pooled buffer set, growing it if this transfer needs more room
batch_src, batch_dst, batch_sizes = (
self._buffer_pool.pop()
if self._buffer_pool
else _new_descriptor_buffers(num_copy_ops)
)
if batch_src.numel() < num_copy_ops:
batch_src, batch_dst, batch_sizes = _new_descriptor_buffers(num_copy_ops)
src = batch_src[:num_copy_ops]
dst = batch_dst[:num_copy_ops]
sizes = batch_sizes[:num_copy_ops]
all_src = src.numpy()
all_dst = dst.numpy()
all_sizes = sizes.numpy()
src_offset = 0
dst_offset = 0
@@ -294,10 +355,6 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
assert dst_offset == num_dst_blocks
assert op_idx == num_copy_ops
batch_src = torch.from_numpy(all_src)
batch_dst = torch.from_numpy(all_dst)
batch_sizes = torch.from_numpy(all_sizes)
stream = self._stream_pool.pop() if self._stream_pool else torch.cuda.Stream()
start_event = (
self._event_pool.pop()
@@ -328,10 +385,10 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
with torch.cuda.stream(stream):
start_event.record(stream)
if num_copy_ops > 0:
ops.swap_blocks_batch(
batch_src,
batch_dst,
batch_sizes,
self._swap_blocks_batch(
src,
dst,
sizes,
is_src_access_order_any=is_src_access_order_any,
)
end_event.record(stream)
@@ -344,6 +401,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
start_event=start_event,
end_event=end_event,
num_bytes=num_transfer_bytes,
batch_src=batch_src,
batch_dst=batch_dst,
batch_sizes=batch_sizes,
)
)
@@ -370,6 +430,9 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self._stream_pool.append(transfer.stream)
self._event_pool.append(transfer.end_event)
self._event_pool.append(transfer.start_event)
self._buffer_pool.append(
(transfer.batch_src, transfer.batch_dst, transfer.batch_sizes)
)
del self._transfer_events[transfer.job_id]
return results
@@ -388,6 +451,7 @@ class SingleDirectionOffloadingHandler(OffloadingHandler):
self._transfer_events.clear()
self._stream_pool.clear()
self._event_pool.clear()
self._buffer_pool.clear()
self.src_tensors.clear()
self.dst_tensors.clear()
if self._mmap_region is not None:
@@ -0,0 +1,74 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Triton kernel + tuned constants for the ``swap_blocks_batch`` fast path."""
from __future__ import annotations
import torch
from vllm import _custom_ops as ops
from vllm.triton_utils import tl, triton
# Constants tuned empirically on H100 (PCIe Gen5):
# NUM_SMS - smallest SM slice within 5% of peak bandwidth at the
# 8-32 KB block sizes that matter in practice
# THRESHOLD_BYTES - max payload per descriptor where Triton beats DMA; above
# this the C++ cuMemcpyBatchAsync path takes the lead
# MIN_N - minimum batch size where Triton's per-launch cost is
# amortized; below this DMA wins
NUM_SMS = 12
THRESHOLD_BYTES = 28 * 1024
MIN_N = 16
@triton.jit
def _swap_blocks_kernel(
src_addrs,
dst_addrs,
sizes,
n_jobs, # type: ignore[name-defined]
BYTES_PER_CHUNK: tl.constexpr, # type: ignore[name-defined]
):
pid = tl.program_id(0)
num_progs = tl.num_programs(0)
WORDS_PER_CHUNK: tl.constexpr = BYTES_PER_CHUNK // 8
offsets = tl.arange(0, WORDS_PER_CHUNK)
job = pid
while job < n_jobs:
src = tl.load(src_addrs + job).to(tl.pointer_type(tl.int64))
dst = tl.load(dst_addrs + job).to(tl.pointer_type(tl.int64))
words = tl.load(sizes + job) // 8
for start in range(0, words, WORDS_PER_CHUNK):
idx = start + offsets
mask = idx < words
data = tl.load(src + idx, mask=mask, other=0)
tl.store(dst + idx, data, mask=mask)
job += num_progs
def swap_blocks_batch(
src_addrs: torch.Tensor,
dst_addrs: torch.Tensor,
sizes: torch.Tensor,
is_src_access_order_any: bool = False,
*,
bytes_per_chunk: int,
) -> None:
"""Triton implementation of ``swap_blocks_batch`` for small CPU->GPU batches."""
n = src_addrs.numel()
# Too few descriptors to amortize Triton's launch cost.
if n < MIN_N:
ops.swap_blocks_batch(
src_addrs,
dst_addrs,
sizes,
is_src_access_order_any=is_src_access_order_any,
)
return
_swap_blocks_kernel[(min(NUM_SMS, n),)](
src_addrs.to("cuda", non_blocking=True),
dst_addrs.to("cuda", non_blocking=True),
sizes.to("cuda", non_blocking=True),
n,
BYTES_PER_CHUNK=bytes_per_chunk,
)