131 lines
3.9 KiB
Python
131 lines
3.9 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
"""DMA copy backend for GPU<->CPU block transfers."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import queue
|
|
import threading
|
|
|
|
import torch
|
|
|
|
from vllm.logger import init_logger
|
|
from vllm.platforms import current_platform
|
|
from vllm.v1.simple_kv_offload.cuda_mem_ops import (
|
|
CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
|
|
CU_MEMCPY_SRC_ACCESS_ORDER_STREAM,
|
|
BatchMemcpyParams,
|
|
build_params,
|
|
copy_blocks,
|
|
)
|
|
|
|
logger = init_logger(__name__)
|
|
|
|
|
|
class DmaCopyBackend:
|
|
"""cuMemcpyBatchAsync copy backend (background thread)."""
|
|
|
|
def __init__(self) -> None:
|
|
self._store_params: BatchMemcpyParams | None = None
|
|
self._load_params: BatchMemcpyParams | None = None
|
|
self._load_stream: torch.cuda.Stream | None = None
|
|
self._store_stream: torch.cuda.Stream | None = None
|
|
self._queue: queue.SimpleQueue | None = None
|
|
self._thread: threading.Thread | None = None
|
|
self._shutdown: bool = False
|
|
|
|
def init(
|
|
self,
|
|
gpu_caches: dict[str, torch.Tensor],
|
|
cpu_caches: dict[str, torch.Tensor],
|
|
device: torch.device,
|
|
load_stream: torch.cuda.Stream,
|
|
store_stream: torch.cuda.Stream,
|
|
) -> None:
|
|
self._load_stream = load_stream
|
|
self._store_stream = store_stream
|
|
|
|
# Stores read the live KV cache -> STREAM (paired with the compute-done
|
|
# wait in get_finished); loads read stable pinned host memory -> ANY.
|
|
self._store_params = build_params(
|
|
gpu_caches,
|
|
cpu_caches,
|
|
store_stream,
|
|
src_access_order=CU_MEMCPY_SRC_ACCESS_ORDER_STREAM,
|
|
)
|
|
self._load_params = build_params(
|
|
cpu_caches,
|
|
gpu_caches,
|
|
load_stream,
|
|
src_access_order=CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
|
|
)
|
|
|
|
self._queue = queue.SimpleQueue()
|
|
self._thread = threading.Thread(
|
|
target=self._copy_loop,
|
|
args=(self._queue, device, load_stream, store_stream),
|
|
daemon=True,
|
|
)
|
|
self._thread.start()
|
|
|
|
def launch_copy(
|
|
self,
|
|
src_blocks: list[int],
|
|
dst_blocks: list[int],
|
|
is_store: bool,
|
|
event_idx: int,
|
|
events_list: list[tuple[int, torch.Event]],
|
|
wait_event: torch.Event | None = None,
|
|
) -> None:
|
|
params = self._store_params if is_store else self._load_params
|
|
assert params is not None and self._queue is not None
|
|
self._queue.put(
|
|
(
|
|
src_blocks,
|
|
dst_blocks,
|
|
params,
|
|
is_store,
|
|
event_idx,
|
|
events_list,
|
|
wait_event,
|
|
)
|
|
)
|
|
|
|
def shutdown(self) -> None:
|
|
if self._shutdown:
|
|
return
|
|
self._shutdown = True
|
|
if self._queue is not None:
|
|
self._queue.put(None)
|
|
if self._thread is not None:
|
|
self._thread.join(timeout=5.0)
|
|
|
|
@staticmethod
|
|
def _copy_loop(
|
|
q: queue.SimpleQueue,
|
|
device: torch.device,
|
|
load_stream: torch.cuda.Stream,
|
|
store_stream: torch.cuda.Stream,
|
|
) -> None:
|
|
current_platform.set_device(device)
|
|
while True:
|
|
item = q.get()
|
|
if item is None:
|
|
return
|
|
(
|
|
src_blocks,
|
|
dst_blocks,
|
|
params,
|
|
is_store,
|
|
event_idx,
|
|
events_list,
|
|
wait_event,
|
|
) = item
|
|
stream = store_stream if is_store else load_stream
|
|
if wait_event is not None:
|
|
stream.wait_event(wait_event)
|
|
copy_blocks(src_blocks, dst_blocks, params)
|
|
event = torch.Event()
|
|
event.record(stream)
|
|
events_list.append((event_idx, event))
|