[Core] Enhance cpu<->gpu sync checking to include paged async copies (#53491)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Nick Hill
2026-09-08 17:26:21 -07:00
committed by GitHub
parent 9e137d8d23
commit 4aadb0f14d
5 changed files with 332 additions and 14 deletions
+146
View File
@@ -158,3 +158,149 @@ def test_without_env_set(monkeypatch):
with_gpu_sync_check(_no_sync)()
with_gpu_sync_check(_causes_sync)()
def _pageable_h2d_nonblocking():
# Pageable source: the "async" copy is staged through pageable memory.
torch.zeros(4).to("cuda", non_blocking=True)
def _pinned_h2d_nonblocking():
# Pinned, contiguous, same dtype: genuinely asynchronous.
torch.zeros(4).pin_memory().to("cuda", non_blocking=True)
def _noncontiguous_h2d_nonblocking():
# Pinned but gapped layout (strided slice): staged through a pageable temp.
torch.zeros(8).pin_memory()[::2].to("cuda", non_blocking=True)
def _transposed_h2d_nonblocking():
# Dense permuted layout: copied with pitched cudaMemcpy2DAsync, stays async.
torch.zeros(4, 4).pin_memory().t().to("cuda", non_blocking=True)
def _dtype_converting_h2d_nonblocking():
# Pinned and contiguous: the conversion runs GPU-side, staying async.
torch.zeros(4, dtype=torch.float64).pin_memory().to(
"cuda", dtype=torch.float32, non_blocking=True
)
def _pageable_h2d_via_cuda():
torch.zeros(4).cuda(non_blocking=True)
def _pageable_h2d_via_copy_():
torch.zeros(4, device="cuda").copy_(torch.zeros(4), non_blocking=True)
def _d2h_via_to():
# Genuinely asynchronous: torch allocates the destination pinned.
torch.zeros(4, device="cuda").to("cpu", non_blocking=True)
def _pageable_d2h_via_copy_():
torch.zeros(4).copy_(torch.zeros(4, device="cuda"), non_blocking=True)
def _pinned_d2h_via_copy_():
# Pinned, contiguous, same dtype: genuinely asynchronous.
torch.zeros(4).pin_memory().copy_(torch.zeros(4, device="cuda"), non_blocking=True)
def _noncontiguous_d2h_via_copy_():
# Pinned but gapped (strided slice) destination.
torch.zeros(8).pin_memory()[::2].copy_(
torch.zeros(4, device="cuda"), non_blocking=True
)
def _transposed_d2h_via_copy_():
# Dense permuted destination: pitched cudaMemcpy2DAsync, stays async.
torch.zeros(4, 4).pin_memory().t().copy_(
torch.zeros(4, 4, device="cuda"), non_blocking=True
)
def _empty_h2d_nonblocking():
# Empty (e.g. first-step penalties): no CUDA call is issued at all.
torch.zeros(0).to("cuda", non_blocking=True)
def _dtype_converting_d2h_via_copy_():
# Pinned and contiguous: the conversion runs GPU-side, staying async.
torch.zeros(4).pin_memory().copy_(
torch.zeros(4, dtype=torch.float64, device="cuda"), non_blocking=True
)
@pytest.mark.parametrize("mode", ["warn", "error"])
@pytest.mark.parametrize(
"fn",
[
_pageable_h2d_nonblocking,
_noncontiguous_h2d_nonblocking,
_pageable_h2d_via_cuda,
_pageable_h2d_via_copy_,
_pageable_d2h_via_copy_,
_noncontiguous_d2h_via_copy_,
],
)
@create_new_process_for_each_test()
def test_implicit_copy_sync_detected(monkeypatch, mode, fn):
"""`non_blocking=True` CPU<->CUDA copies with a pageable or
non-densely-laid-out CPU tensor may block the host without tripping
torch's sync debug mode; the `Tensor.to`/`cuda`/`copy_` wrappers must
flag them.
"""
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", mode)
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
if mode == "error":
with pytest.raises(RuntimeError, match="Implicit GPU<->CPU sync"):
with_gpu_sync_check(fn)()
else:
with pytest.warns(UserWarning, match="Implicit GPU<->CPU sync"):
with_gpu_sync_check(fn)()
@create_new_process_for_each_test()
def test_genuinely_async_transfers_pass(monkeypatch):
"""Pinned, densely laid out CPU tensors make `non_blocking=True` truly
asynchronous in both directions -- even with a dtype conversion, which
runs GPU-side, and even permuted (e.g. transposed), which uses pitched
cudaMemcpy2D/3DAsync; D2H `Tensor.to` allocates a pinned destination.
None of these may be flagged."""
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
for fn in (
_pinned_h2d_nonblocking,
_pinned_d2h_via_copy_,
_d2h_via_to,
_dtype_converting_h2d_nonblocking,
_dtype_converting_d2h_via_copy_,
_transposed_h2d_nonblocking,
_transposed_d2h_via_copy_,
_empty_h2d_nonblocking,
):
fn() # Warm up outside the checked region.
with_gpu_sync_check(fn)()
@create_new_process_for_each_test()
def test_implicit_copy_sync_can_be_allowed(monkeypatch):
"""`gpu_sync_allowed()` must exempt implicit copy syncs too."""
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
gsd._install_copy_checkers()
def allowed_pageable_copies():
with gpu_sync_allowed():
_pageable_h2d_nonblocking()
_pageable_d2h_via_copy_()
with_gpu_sync_check(allowed_pageable_copies)()
@@ -135,7 +135,11 @@ class ExampleConnector(KVConnectorBase_V1):
slot_mapping (torch.Tensor): the slot mapping. In shape
[num_tokens].
"""
slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True)
# Deliberate bulk transfers in this reference connector.
with gpu_sync_allowed():
slot_mapping = slot_mapping.to(
dst_kv_cache_layer.device, non_blocking=True
)
if isinstance(attn_metadata, MLACommonMetadata):
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
@@ -177,8 +181,10 @@ class ExampleConnector(KVConnectorBase_V1):
filename = self._generate_filename_debug(
layer_name, request.token_ids, request.mm_hashes
)
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
# Deliberate bulk load in this reference connector.
with gpu_sync_allowed():
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
if isinstance(attn_metadata, dict):
inject_kv_into_layer(
kv_cache_layer,
@@ -224,7 +230,9 @@ class ExampleConnector(KVConnectorBase_V1):
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
"""
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
# Deliberate bulk transfer in this reference connector.
with gpu_sync_allowed():
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
if isinstance(attn_metadata, MLACommonMetadata):
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
+5 -3
View File
@@ -642,9 +642,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"VLLM_TRITON_USE_TD": lambda: {"1": True, "0": False}.get(
os.getenv("VLLM_TRITON_USE_TD", "").strip()
),
# If set, enable PyTorch's GPU<->CPU synchronization debug mode around
# the worker's `execute_model` and `sample_tokens` calls. Valid values
# are "warn" (print a warning on each sync) or "error" (raise on sync).
# If set, enable GPU<->CPU synchronization checking around the worker's
# `execute_model` and `sample_tokens` calls, via PyTorch's sync debug mode
# plus wrappers flagging `non_blocking` CPU<->CUDA copies that silently
# block the host (CPU tensors that are not pinned or not densely laid out).
# Valid values are "warn" (warn on each sync) or "error" (raise on sync).
# Unset disables the check. See `torch.cuda.set_sync_debug_mode`.
"VLLM_GPU_SYNC_CHECK": env_with_choices(
"VLLM_GPU_SYNC_CHECK", None, ["warn", "error"]
+10 -7
View File
@@ -3,6 +3,7 @@
"""UVA-based CPU offloading using Unified Virtual Addressing."""
from collections.abc import Generator
from contextlib import nullcontext
import torch
import torch.nn as nn
@@ -11,6 +12,7 @@ from torch.func import functional_call
import vllm.envs as envs
from vllm.logger import init_logger
from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.mem_utils import format_gib
from vllm.utils.platform_utils import is_uva_available
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
@@ -126,13 +128,14 @@ class UVAOffloader(BaseOffloader):
def forward(*args, **kwargs):
module.forward = original_forward
device_state = {
# here we blindly call `to(device)`
# if the parameter is already on the device,
# it will be a no-op
k: v.to(device, non_blocking=True)
for k, v in module.state_dict().items()
}
with nullcontext() if self.pin_memory else gpu_sync_allowed():
device_state = {
# here we blindly call `to(device)`
# if the parameter is already on the device,
# it will be a no-op
k: v.to(device, non_blocking=True)
for k, v in module.state_dict().items()
}
# set `tie_weights=False` as tied weights in original model
# become untied when calling .to(device) individually
+159
View File
@@ -7,6 +7,12 @@
is a failure. The scoping lives in ContextVars, which are per-thread and
per-asyncio-task, so an allow region opened on one thread is invisible to
every other one.
torch's mode only instruments explicit synchronizing calls, so it misses
`non_blocking=True` CPU<->CUDA copies that the CUDA driver silently stages
through pageable memory (when the CPU tensor is not pinned or not densely
laid out). Those are caught by the wrappers on `Tensor.to`/`Tensor.cuda`/
`Tensor.copy_` installed by `_install_copy_checkers()`.
"""
import functools
@@ -54,6 +60,7 @@ def enable_gpu_sync_check() -> None:
global _sync_check_enabled
_sync_check_enabled = True
_install_compile_time_sync_suppressors()
_install_copy_checkers()
_arm_lock = threading.Lock()
@@ -197,6 +204,158 @@ def _install_compile_time_sync_suppressors() -> None:
pass
# ---------------------------------------------------------------------------
# Implicit sync detection in `non_blocking` CPU<->CUDA copies.
#
# A `non_blocking=True` copy between CPU and CUDA still blocks the host when
# the CPU side is not pinned or not densely laid out (gapped strided slices
# stage through a pageable temp): D2H copies into such memory wait for the
# stream, and H2D copies from it do too beyond a driver size threshold.
# None of that goes through an API torch's sync debug mode instruments, so
# wrap the Python-level transfer entry points and flag such copies on threads
# in a checked region.
IMPLICIT_SYNC_ERROR_MESSAGE = (
"Implicit GPU<->CPU sync detected - use a pinned, densely laid out CPU "
"tensor, or wrap with gpu_sync_allowed()"
)
# Captured at import; `_install_copy_checkers` replaces these attributes.
_torch_to = torch.Tensor.to
_torch_cuda = torch.Tensor.cuda
_torch_copy_ = torch.Tensor.copy_
# torch's own parser for `Tensor.to`'s overloaded signature, returning
# (device, dtype, non_blocking, memory_format) (also used by torch._dynamo).
_parse_to = getattr(torch._C._nn, "_parse_to", None)
def _active_check_mode() -> str | None:
"""The mode the calling thread is being checked in, if any."""
if not _sync_check_enabled or _allow_depth.get() or torch.compiler.is_compiling():
return None
return _checking.get()
def _is_dense(t: torch.Tensor) -> bool:
"""Whether `t`'s elements fill a contiguous storage range, possibly
permuted (e.g. a transposed matrix). Such layouts can be copied with
pitched cudaMemcpy2D/3DAsync directly from/to pinned memory, whereas
gapped layouts (e.g. strided slices) stage through a pageable temp."""
if t.is_contiguous():
# The common case, answered by a memoized TensorImpl flag.
return True
expected_stride = 1
for stride, size in sorted(zip(t.stride(), t.shape)):
if size <= 1:
continue
if stride != expected_stride:
return False
expected_stride *= size
return True
def _cpu_copy_stall_reason(
cpu_tensor: torch.Tensor, device: torch.device | None
) -> str | None:
"""Why a `non_blocking` copy between `cpu_tensor` and a CUDA tensor would
silently block the host, or None if it is genuinely asynchronous.
`device` describes the other side of the copy; a non-CUDA `device` means
the copy does not involve the GPU at all.
"""
if device is None or device.type != "cuda" or cpu_tensor.device.type != "cpu":
return None
if cpu_tensor.numel() == 0:
return None # Nothing to copy; torch issues no CUDA call at all.
if not _is_dense(cpu_tensor):
return "the CPU tensor is not densely laid out"
if not cpu_tensor.is_pinned():
return "the CPU tensor is not pinned"
return None
def _report_implicit_sync(op: str, direction: str, reason: str, mode: str) -> None:
msg = (
f"Tensor.{op}: a non_blocking {direction} copy that may be silently "
f"synchronous ({reason}). {IMPLICIT_SYNC_ERROR_MESSAGE}"
)
if mode == "error":
raise RuntimeError(msg)
warnings.warn(msg, stacklevel=3)
def _checked_to(self, *args, **kwargs):
# Only H2D is checked: D2H `to` allocates a pinned destination in
# supported torch versions, so it is genuinely asynchronous.
if (mode := _active_check_mode()) is not None and _parse_to is not None:
try:
device, _, non_blocking, _ = _parse_to(*args, **kwargs)
except Exception:
pass # Invalid args; let the original call raise.
else:
if non_blocking and (reason := _cpu_copy_stall_reason(self, device)):
_report_implicit_sync("to", "H2D", reason, mode)
return _torch_to(self, *args, **kwargs)
def _checked_cuda(self, *args, **kwargs):
# Signature: cuda(device=None, non_blocking=False, memory_format=None).
if (mode := _active_check_mode()) is not None:
device = args[0] if args else kwargs.get("device")
non_blocking = args[1] if len(args) > 1 else kwargs.get("non_blocking", False)
if non_blocking:
if device is None:
device = torch.device("cuda")
elif isinstance(device, int):
device = torch.device("cuda", device)
elif not isinstance(device, torch.device):
device = torch.device(device)
if reason := _cpu_copy_stall_reason(self, device):
_report_implicit_sync("cuda", "H2D", reason, mode)
return _torch_cuda(self, *args, **kwargs)
def _checked_copy_(self, *args, **kwargs):
# Signature: copy_(src, non_blocking=False).
if (
(mode := _active_check_mode()) is not None
and args
and isinstance(src := args[0], torch.Tensor)
):
non_blocking = args[1] if len(args) > 1 else kwargs.get("non_blocking", False)
if non_blocking:
if self.device.type == "cuda":
reason = _cpu_copy_stall_reason(src, self.device)
direction = "H2D"
else:
reason = _cpu_copy_stall_reason(self, src.device)
direction = "D2H"
if reason:
_report_implicit_sync("copy_", direction, reason, mode)
return _torch_copy_(self, *args, **kwargs)
_copy_checkers_installed: bool = False
def _install_copy_checkers() -> None:
"""Wrap the Python-level CPU<->CUDA transfer entry points with the checks
above.
The patch is process-global, but it only reports on threads inside a
checked region, matching the scoping of the torch-level check. Copies
issued from C++ (e.g. inside compiled graphs) bypass it.
"""
global _copy_checkers_installed
if _copy_checkers_installed:
return
_copy_checkers_installed = True
torch.Tensor.to = _checked_to # type: ignore[method-assign]
torch.Tensor.cuda = _checked_cuda # type: ignore[method-assign]
torch.Tensor.copy_ = _checked_copy_ # type: ignore[method-assign]
def gpu_sync_allowed(first_only: bool = False):
"""Allow GPU<->CPU syncs inside the `with` block, on this thread only.