[Bugfix][KVConnector] Fix SimpleCPUOffloadConnector GPU->CPU store race (#46278)

This commit is contained in:
Saddss
2026-06-22 13:08:47 -07:00
committed by GitHub
parent fbf520cf3a
commit 82ede09a5a
4 changed files with 244 additions and 16 deletions
+183
View File
@@ -0,0 +1,183 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Worker-side unit tests for SimpleCPUOffloadConnector.
Covers the GPU->CPU store cross-stream synchronization: the store copy must be
ordered after the compute stream that writes the KV blocks, otherwise it can
read partially written / stale blocks and silently corrupt the CPU cache.
"""
from __future__ import annotations
import time
import pytest
import torch
from vllm.platforms import current_platform
if not current_platform.is_cuda_alike():
pytest.skip("Requires CUDA or ROCm", allow_module_level=True)
from vllm.v1.simple_kv_offload.copy_backend import DmaCopyBackend
from vllm.v1.simple_kv_offload.cuda_mem_ops import (
CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
CU_MEMCPY_SRC_ACCESS_ORDER_STREAM,
build_params,
pin_tensor,
)
from vllm.v1.simple_kv_offload.metadata import SimpleCPUOffloadMetadata
from vllm.v1.simple_kv_offload.worker import SimpleCPUOffloadWorker
NUM_BLOCKS = 64
BLOCK_BYTES = 4096
ITERS = 30
# Keep the compute stream busy so the KV write lands late; this makes the
# store-vs-compute race deterministic instead of timing-dependent.
SLEEP_CYCLES = 50_000_000
def _make_backend() -> tuple[DmaCopyBackend, torch.Tensor, torch.Tensor]:
gpu = {"k": torch.zeros((NUM_BLOCKS, BLOCK_BYTES), dtype=torch.int8, device="cuda")}
cpu = {"k": torch.zeros((NUM_BLOCKS, BLOCK_BYTES), dtype=torch.int8, device="cpu")}
pin_tensor(cpu["k"])
low_pri, _ = torch.cuda.Stream.priority_range()
backend = DmaCopyBackend()
backend.init(
gpu,
cpu,
gpu["k"].device,
torch.cuda.Stream(priority=low_pri),
torch.cuda.Stream(priority=low_pri),
)
return backend, gpu["k"], cpu["k"]
def _drive_store(
backend: DmaCopyBackend,
gpu: torch.Tensor,
cpu: torch.Tensor,
*,
with_barrier: bool,
) -> int:
"""Run ITERS store cycles; return how many landed corrupted in the CPU pool.
Each cycle writes a unique value on a compute stream (after a deliberate
delay) and then issues the GPU->CPU store. The store is issued *after* the
write in host program order, mirroring the connector's deferred-store
assumption. Only the compute-done event creates a real device-side
happens-before edge.
"""
block_ids = list(range(gpu.shape[0]))
compute_stream = torch.cuda.Stream()
corrupt = 0
for it in range(ITERS):
val = (it % 126) + 1 # 1..126; distinct from the zero-initialized pool
with torch.cuda.stream(compute_stream):
torch.cuda._sleep(SLEEP_CYCLES)
gpu.fill_(val)
wait_event = None
if with_barrier:
wait_event = torch.Event()
wait_event.record(compute_stream)
store_events: list[tuple[int, torch.Event]] = []
backend.launch_copy(
block_ids,
block_ids,
is_store=True,
event_idx=it,
events_list=store_events,
wait_event=wait_event,
)
deadline = time.time() + 10.0
while not store_events and time.time() < deadline:
time.sleep(0.0005)
assert store_events, "background copy was never enqueued"
store_events[0][1].synchronize()
if int((cpu[:, 0].to(torch.int32) != val).sum().item()):
corrupt += 1
return corrupt
def test_store_orders_after_compute_write():
"""The store must wait for the compute event; without it, it races.
Asserts both directions so the test is self-validating: the no-barrier
control must actually corrupt (proving the race window is exercised), and
the fixed path with the compute-done event must be clean.
"""
backend, gpu, cpu = _make_backend()
try:
control = _drive_store(backend, gpu, cpu, with_barrier=False)
fixed = _drive_store(backend, gpu, cpu, with_barrier=True)
finally:
backend.shutdown()
assert control > 0, (
"no-barrier store did not race the compute write; the test no longer "
"exercises the hazard it is meant to guard"
)
assert fixed == 0, f"store raced compute even with the barrier: {fixed} corrupt"
class _RecordingBackend:
"""Captures launch_copy calls without touching the GPU."""
def __init__(self) -> None:
self.calls: list[dict] = []
def launch_copy(
self,
src_blocks,
dst_blocks,
is_store,
event_idx,
events_list,
wait_event=None,
) -> None:
self.calls.append({"is_store": is_store, "wait_event": wait_event})
def test_get_finished_passes_wait_event_for_store_only():
"""get_finished gates stores on a compute-done event but not loads."""
worker = SimpleCPUOffloadWorker(
vllm_config=None, kv_cache_config=None, cpu_capacity_bytes=0
)
recording = _RecordingBackend()
worker._backend = recording
worker._connector_metadata = SimpleCPUOffloadMetadata(
load_event=0,
load_gpu_blocks=[0],
load_cpu_blocks=[0],
store_event=1,
store_gpu_blocks=[1],
store_cpu_blocks=[1],
)
worker.get_finished(set())
store_calls = [c for c in recording.calls if c["is_store"]]
load_calls = [c for c in recording.calls if not c["is_store"]]
assert len(store_calls) == 1
assert len(load_calls) == 1
assert isinstance(store_calls[0]["wait_event"], torch.Event)
assert load_calls[0]["wait_event"] is None
def test_build_params_src_access_order():
"""build_params defaults to ANY and honors an explicit STREAM override."""
gpu = {"k": torch.zeros((4, 64), dtype=torch.int8, device="cuda")}
cpu = {"k": torch.zeros((4, 64), dtype=torch.int8, device="cpu")}
stream = torch.cuda.Stream()
default = build_params(gpu, cpu, stream)
assert default.attrs.srcAccessOrder == CU_MEMCPY_SRC_ACCESS_ORDER_ANY
ordered = build_params(
gpu, cpu, stream, src_access_order=CU_MEMCPY_SRC_ACCESS_ORDER_STREAM
)
assert ordered.attrs.srcAccessOrder == CU_MEMCPY_SRC_ACCESS_ORDER_STREAM
+38 -5
View File
@@ -12,6 +12,8 @@ import torch
from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.v1.simple_kv_offload.cuda_mem_ops import (
CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
CU_MEMCPY_SRC_ACCESS_ORDER_STREAM,
BatchMemcpyParams,
build_params,
copy_blocks,
@@ -43,8 +45,20 @@ class DmaCopyBackend:
self._load_stream = load_stream
self._store_stream = store_stream
self._store_params = build_params(gpu_caches, cpu_caches, store_stream)
self._load_params = build_params(cpu_caches, gpu_caches, load_stream)
# Stores read the live KV cache -> STREAM (paired with the compute-done
# wait in get_finished); loads read stable pinned host memory -> ANY.
self._store_params = build_params(
gpu_caches,
cpu_caches,
store_stream,
src_access_order=CU_MEMCPY_SRC_ACCESS_ORDER_STREAM,
)
self._load_params = build_params(
cpu_caches,
gpu_caches,
load_stream,
src_access_order=CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
)
self._queue = queue.SimpleQueue()
self._thread = threading.Thread(
@@ -61,11 +75,20 @@ class DmaCopyBackend:
is_store: bool,
event_idx: int,
events_list: list[tuple[int, torch.Event]],
wait_event: torch.Event | None = None,
) -> None:
params = self._store_params if is_store else self._load_params
assert params is not None and self._queue is not None
self._queue.put(
(src_blocks, dst_blocks, params, is_store, event_idx, events_list)
(
src_blocks,
dst_blocks,
params,
is_store,
event_idx,
events_list,
wait_event,
)
)
def shutdown(self) -> None:
@@ -89,9 +112,19 @@ class DmaCopyBackend:
item = q.get()
if item is None:
return
src_blocks, dst_blocks, params, is_store, event_idx, events_list = item
copy_blocks(src_blocks, dst_blocks, params)
(
src_blocks,
dst_blocks,
params,
is_store,
event_idx,
events_list,
wait_event,
) = item
stream = store_stream if is_store else load_stream
if wait_event is not None:
stream.wait_event(wait_event)
copy_blocks(src_blocks, dst_blocks, params)
event = torch.Event()
event.record(stream)
events_list.append((event_idx, event))
+10 -6
View File
@@ -13,6 +13,12 @@ from vllm.platforms import current_platform
logger = init_logger(__name__)
# CUmemcpySrcAccessOrder values (CUDA driver API). STREAM(1): source read in
# stream order, safe when the source may still be written. ANY(3): source may
# be read early, only safe for a stable source (e.g. pinned host memory).
CU_MEMCPY_SRC_ACCESS_ORDER_STREAM = 1
CU_MEMCPY_SRC_ACCESS_ORDER_ANY = 3
def pin_tensor(tensor: torch.Tensor) -> None:
"""Pin a CPU tensor via cudaHostRegister.
@@ -106,8 +112,8 @@ class BatchMemcpyParams(NamedTuple):
dst_bases: np.ndarray # [num_layers] uint64
bpb: np.ndarray # [num_layers] uint64 — bytes per block
num_layers: int
# CUDA only: one attributes entry with srcAccessOrder=ANY. Unused on
# ROCm (7.2.1 or 7.2.2) because the current runtime rejects numAttrs > 0.
# CUDA only: one attributes entry carrying srcAccessOrder. Unused on ROCm
# (7.2.1 or 7.2.2) because the current runtime rejects numAttrs > 0.
attrs: _CUmemcpyAttributes
attrs_idx: ctypes.c_size_t
# NOTE: cuMemcpyBatchAsync_v2() removed fail_idx field, but we use
@@ -120,6 +126,7 @@ def build_params(
src_caches: dict[str, torch.Tensor],
dst_caches: dict[str, torch.Tensor],
stream: torch.cuda.Stream,
src_access_order: int = CU_MEMCPY_SRC_ACCESS_ORDER_ANY,
) -> BatchMemcpyParams:
global _batch_memcpy_fn
if _batch_memcpy_fn is None:
@@ -137,10 +144,7 @@ def build_params(
dst_bases.append(d.data_ptr())
bpb.append(s_bpb)
# ``srcAccessOrder=3`` == CU_MEMCPY_SRC_ACCESS_ORDER_ANY /
# hipMemcpySrcAccessOrderAny. See
# https://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__MEM.html#group__CUDA__MEM_1g6f1ff58e3065df3eb4b573dba77ad31f # noqa: E501
attrs = _CUmemcpyAttributes(srcAccessOrder=3)
attrs = _CUmemcpyAttributes(srcAccessOrder=src_access_order)
return BatchMemcpyParams(
src_bases=np.array(src_bases, dtype=np.uint64),
+13 -5
View File
@@ -57,6 +57,10 @@ class SimpleCPUOffloadWorker:
# Metadata for the current step
self._connector_metadata: SimpleCPUOffloadMetadata | None = None
# Compute-done event recorded before each store; reused across steps
# (get_finished runs once per step, copy queue is FIFO).
self._store_compute_done: torch.Event | None = None
# Pending event index sets, populated in bind_connector_metadata
self._pending_load_event_indices: set[int] = set()
self._pending_store_event_indices: set[int] = set()
@@ -206,9 +210,11 @@ class SimpleCPUOffloadWorker:
) -> tuple[set[str] | None, set[str] | None]:
"""Submit transfers and report completed events to the scheduler.
Called after model execution. The manager only schedules stores for
blocks whose KV data is confirmed computed, so we launch both loads
and stores immediately — no deferral or cross-stream sync needed.
Stores (GPU->CPU) read the live KV cache, which the compute stream may
still be writing under v1 overlapped execution, so they are ordered
after a compute-done event recorded on the current stream. Loads
(CPU->GPU) read stable pinned host memory and launch immediately. See
#45704 for the bug and #39306 for the srcAccessOrder rationale.
Returns:
tuple of (finished_sending, finished_recving).
@@ -218,7 +224,6 @@ class SimpleCPUOffloadWorker:
# (1) Submit transfers
metadata = self._connector_metadata
if metadata is not None:
# Launch loads (CPU->GPU).
if metadata.load_cpu_blocks:
self._backend.launch_copy(
metadata.load_cpu_blocks,
@@ -227,14 +232,17 @@ class SimpleCPUOffloadWorker:
event_idx=metadata.load_event,
events_list=self._load_events,
)
# Launch stores (GPU->CPU).
if metadata.store_gpu_blocks:
if self._store_compute_done is None:
self._store_compute_done = torch.Event()
self._store_compute_done.record(torch.cuda.current_stream())
self._backend.launch_copy(
metadata.store_gpu_blocks,
metadata.store_cpu_blocks,
is_store=True,
event_idx=metadata.store_event,
events_list=self._store_events,
wait_event=self._store_compute_done,
)
# (2) Track completed transfer events