[Core] Enhance cpu<->gpu sync checking to include paged async copies (#53491)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -158,3 +158,149 @@ def test_without_env_set(monkeypatch):
|
||||
|
||||
with_gpu_sync_check(_no_sync)()
|
||||
with_gpu_sync_check(_causes_sync)()
|
||||
|
||||
|
||||
def _pageable_h2d_nonblocking():
|
||||
# Pageable source: the "async" copy is staged through pageable memory.
|
||||
torch.zeros(4).to("cuda", non_blocking=True)
|
||||
|
||||
|
||||
def _pinned_h2d_nonblocking():
|
||||
# Pinned, contiguous, same dtype: genuinely asynchronous.
|
||||
torch.zeros(4).pin_memory().to("cuda", non_blocking=True)
|
||||
|
||||
|
||||
def _noncontiguous_h2d_nonblocking():
|
||||
# Pinned but gapped layout (strided slice): staged through a pageable temp.
|
||||
torch.zeros(8).pin_memory()[::2].to("cuda", non_blocking=True)
|
||||
|
||||
|
||||
def _transposed_h2d_nonblocking():
|
||||
# Dense permuted layout: copied with pitched cudaMemcpy2DAsync, stays async.
|
||||
torch.zeros(4, 4).pin_memory().t().to("cuda", non_blocking=True)
|
||||
|
||||
|
||||
def _dtype_converting_h2d_nonblocking():
|
||||
# Pinned and contiguous: the conversion runs GPU-side, staying async.
|
||||
torch.zeros(4, dtype=torch.float64).pin_memory().to(
|
||||
"cuda", dtype=torch.float32, non_blocking=True
|
||||
)
|
||||
|
||||
|
||||
def _pageable_h2d_via_cuda():
|
||||
torch.zeros(4).cuda(non_blocking=True)
|
||||
|
||||
|
||||
def _pageable_h2d_via_copy_():
|
||||
torch.zeros(4, device="cuda").copy_(torch.zeros(4), non_blocking=True)
|
||||
|
||||
|
||||
def _d2h_via_to():
|
||||
# Genuinely asynchronous: torch allocates the destination pinned.
|
||||
torch.zeros(4, device="cuda").to("cpu", non_blocking=True)
|
||||
|
||||
|
||||
def _pageable_d2h_via_copy_():
|
||||
torch.zeros(4).copy_(torch.zeros(4, device="cuda"), non_blocking=True)
|
||||
|
||||
|
||||
def _pinned_d2h_via_copy_():
|
||||
# Pinned, contiguous, same dtype: genuinely asynchronous.
|
||||
torch.zeros(4).pin_memory().copy_(torch.zeros(4, device="cuda"), non_blocking=True)
|
||||
|
||||
|
||||
def _noncontiguous_d2h_via_copy_():
|
||||
# Pinned but gapped (strided slice) destination.
|
||||
torch.zeros(8).pin_memory()[::2].copy_(
|
||||
torch.zeros(4, device="cuda"), non_blocking=True
|
||||
)
|
||||
|
||||
|
||||
def _transposed_d2h_via_copy_():
|
||||
# Dense permuted destination: pitched cudaMemcpy2DAsync, stays async.
|
||||
torch.zeros(4, 4).pin_memory().t().copy_(
|
||||
torch.zeros(4, 4, device="cuda"), non_blocking=True
|
||||
)
|
||||
|
||||
|
||||
def _empty_h2d_nonblocking():
|
||||
# Empty (e.g. first-step penalties): no CUDA call is issued at all.
|
||||
torch.zeros(0).to("cuda", non_blocking=True)
|
||||
|
||||
|
||||
def _dtype_converting_d2h_via_copy_():
|
||||
# Pinned and contiguous: the conversion runs GPU-side, staying async.
|
||||
torch.zeros(4).pin_memory().copy_(
|
||||
torch.zeros(4, dtype=torch.float64, device="cuda"), non_blocking=True
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["warn", "error"])
|
||||
@pytest.mark.parametrize(
|
||||
"fn",
|
||||
[
|
||||
_pageable_h2d_nonblocking,
|
||||
_noncontiguous_h2d_nonblocking,
|
||||
_pageable_h2d_via_cuda,
|
||||
_pageable_h2d_via_copy_,
|
||||
_pageable_d2h_via_copy_,
|
||||
_noncontiguous_d2h_via_copy_,
|
||||
],
|
||||
)
|
||||
@create_new_process_for_each_test()
|
||||
def test_implicit_copy_sync_detected(monkeypatch, mode, fn):
|
||||
"""`non_blocking=True` CPU<->CUDA copies with a pageable or
|
||||
non-densely-laid-out CPU tensor may block the host without tripping
|
||||
torch's sync debug mode; the `Tensor.to`/`cuda`/`copy_` wrappers must
|
||||
flag them.
|
||||
"""
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", mode)
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
|
||||
if mode == "error":
|
||||
with pytest.raises(RuntimeError, match="Implicit GPU<->CPU sync"):
|
||||
with_gpu_sync_check(fn)()
|
||||
else:
|
||||
with pytest.warns(UserWarning, match="Implicit GPU<->CPU sync"):
|
||||
with_gpu_sync_check(fn)()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_genuinely_async_transfers_pass(monkeypatch):
|
||||
"""Pinned, densely laid out CPU tensors make `non_blocking=True` truly
|
||||
asynchronous in both directions -- even with a dtype conversion, which
|
||||
runs GPU-side, and even permuted (e.g. transposed), which uses pitched
|
||||
cudaMemcpy2D/3DAsync; D2H `Tensor.to` allocates a pinned destination.
|
||||
None of these may be flagged."""
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
|
||||
for fn in (
|
||||
_pinned_h2d_nonblocking,
|
||||
_pinned_d2h_via_copy_,
|
||||
_d2h_via_to,
|
||||
_dtype_converting_h2d_nonblocking,
|
||||
_dtype_converting_d2h_via_copy_,
|
||||
_transposed_h2d_nonblocking,
|
||||
_transposed_d2h_via_copy_,
|
||||
_empty_h2d_nonblocking,
|
||||
):
|
||||
fn() # Warm up outside the checked region.
|
||||
with_gpu_sync_check(fn)()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_implicit_copy_sync_can_be_allowed(monkeypatch):
|
||||
"""`gpu_sync_allowed()` must exempt implicit copy syncs too."""
|
||||
monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error")
|
||||
monkeypatch.setattr(gsd, "_sync_check_enabled", True)
|
||||
gsd._install_copy_checkers()
|
||||
|
||||
def allowed_pageable_copies():
|
||||
with gpu_sync_allowed():
|
||||
_pageable_h2d_nonblocking()
|
||||
_pageable_d2h_via_copy_()
|
||||
|
||||
with_gpu_sync_check(allowed_pageable_copies)()
|
||||
|
||||
@@ -135,7 +135,11 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
slot_mapping (torch.Tensor): the slot mapping. In shape
|
||||
[num_tokens].
|
||||
"""
|
||||
slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True)
|
||||
# Deliberate bulk transfers in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping = slot_mapping.to(
|
||||
dst_kv_cache_layer.device, non_blocking=True
|
||||
)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
dst_kv_cache_layer = dst_kv_cache_layer.reshape(
|
||||
@@ -177,8 +181,10 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
filename = self._generate_filename_debug(
|
||||
layer_name, request.token_ids, request.mm_hashes
|
||||
)
|
||||
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
|
||||
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
|
||||
# Deliberate bulk load in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"]
|
||||
kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, dict):
|
||||
inject_kv_into_layer(
|
||||
kv_cache_layer,
|
||||
@@ -224,7 +230,9 @@ class ExampleConnector(KVConnectorBase_V1):
|
||||
|
||||
The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA).
|
||||
"""
|
||||
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
|
||||
# Deliberate bulk transfer in this reference connector.
|
||||
with gpu_sync_allowed():
|
||||
slot_mapping = slot_mapping.to(layer.device, non_blocking=True)
|
||||
if isinstance(attn_metadata, MLACommonMetadata):
|
||||
# [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots.
|
||||
return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...]
|
||||
|
||||
+5
-3
@@ -642,9 +642,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"VLLM_TRITON_USE_TD": lambda: {"1": True, "0": False}.get(
|
||||
os.getenv("VLLM_TRITON_USE_TD", "").strip()
|
||||
),
|
||||
# If set, enable PyTorch's GPU<->CPU synchronization debug mode around
|
||||
# the worker's `execute_model` and `sample_tokens` calls. Valid values
|
||||
# are "warn" (print a warning on each sync) or "error" (raise on sync).
|
||||
# If set, enable GPU<->CPU synchronization checking around the worker's
|
||||
# `execute_model` and `sample_tokens` calls, via PyTorch's sync debug mode
|
||||
# plus wrappers flagging `non_blocking` CPU<->CUDA copies that silently
|
||||
# block the host (CPU tensors that are not pinned or not densely laid out).
|
||||
# Valid values are "warn" (warn on each sync) or "error" (raise on sync).
|
||||
# Unset disables the check. See `torch.cuda.set_sync_debug_mode`.
|
||||
"VLLM_GPU_SYNC_CHECK": env_with_choices(
|
||||
"VLLM_GPU_SYNC_CHECK", None, ["warn", "error"]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
"""UVA-based CPU offloading using Unified Virtual Addressing."""
|
||||
|
||||
from collections.abc import Generator
|
||||
from contextlib import nullcontext
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
@@ -11,6 +12,7 @@ from torch.func import functional_call
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
from vllm.utils.mem_utils import format_gib
|
||||
from vllm.utils.platform_utils import is_uva_available
|
||||
from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor
|
||||
@@ -126,13 +128,14 @@ class UVAOffloader(BaseOffloader):
|
||||
|
||||
def forward(*args, **kwargs):
|
||||
module.forward = original_forward
|
||||
device_state = {
|
||||
# here we blindly call `to(device)`
|
||||
# if the parameter is already on the device,
|
||||
# it will be a no-op
|
||||
k: v.to(device, non_blocking=True)
|
||||
for k, v in module.state_dict().items()
|
||||
}
|
||||
with nullcontext() if self.pin_memory else gpu_sync_allowed():
|
||||
device_state = {
|
||||
# here we blindly call `to(device)`
|
||||
# if the parameter is already on the device,
|
||||
# it will be a no-op
|
||||
k: v.to(device, non_blocking=True)
|
||||
for k, v in module.state_dict().items()
|
||||
}
|
||||
|
||||
# set `tie_weights=False` as tied weights in original model
|
||||
# become untied when calling .to(device) individually
|
||||
|
||||
@@ -7,6 +7,12 @@
|
||||
is a failure. The scoping lives in ContextVars, which are per-thread and
|
||||
per-asyncio-task, so an allow region opened on one thread is invisible to
|
||||
every other one.
|
||||
|
||||
torch's mode only instruments explicit synchronizing calls, so it misses
|
||||
`non_blocking=True` CPU<->CUDA copies that the CUDA driver silently stages
|
||||
through pageable memory (when the CPU tensor is not pinned or not densely
|
||||
laid out). Those are caught by the wrappers on `Tensor.to`/`Tensor.cuda`/
|
||||
`Tensor.copy_` installed by `_install_copy_checkers()`.
|
||||
"""
|
||||
|
||||
import functools
|
||||
@@ -54,6 +60,7 @@ def enable_gpu_sync_check() -> None:
|
||||
global _sync_check_enabled
|
||||
_sync_check_enabled = True
|
||||
_install_compile_time_sync_suppressors()
|
||||
_install_copy_checkers()
|
||||
|
||||
|
||||
_arm_lock = threading.Lock()
|
||||
@@ -197,6 +204,158 @@ def _install_compile_time_sync_suppressors() -> None:
|
||||
pass
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Implicit sync detection in `non_blocking` CPU<->CUDA copies.
|
||||
#
|
||||
# A `non_blocking=True` copy between CPU and CUDA still blocks the host when
|
||||
# the CPU side is not pinned or not densely laid out (gapped strided slices
|
||||
# stage through a pageable temp): D2H copies into such memory wait for the
|
||||
# stream, and H2D copies from it do too beyond a driver size threshold.
|
||||
# None of that goes through an API torch's sync debug mode instruments, so
|
||||
# wrap the Python-level transfer entry points and flag such copies on threads
|
||||
# in a checked region.
|
||||
|
||||
IMPLICIT_SYNC_ERROR_MESSAGE = (
|
||||
"Implicit GPU<->CPU sync detected - use a pinned, densely laid out CPU "
|
||||
"tensor, or wrap with gpu_sync_allowed()"
|
||||
)
|
||||
|
||||
# Captured at import; `_install_copy_checkers` replaces these attributes.
|
||||
_torch_to = torch.Tensor.to
|
||||
_torch_cuda = torch.Tensor.cuda
|
||||
_torch_copy_ = torch.Tensor.copy_
|
||||
|
||||
# torch's own parser for `Tensor.to`'s overloaded signature, returning
|
||||
# (device, dtype, non_blocking, memory_format) (also used by torch._dynamo).
|
||||
_parse_to = getattr(torch._C._nn, "_parse_to", None)
|
||||
|
||||
|
||||
def _active_check_mode() -> str | None:
|
||||
"""The mode the calling thread is being checked in, if any."""
|
||||
if not _sync_check_enabled or _allow_depth.get() or torch.compiler.is_compiling():
|
||||
return None
|
||||
return _checking.get()
|
||||
|
||||
|
||||
def _is_dense(t: torch.Tensor) -> bool:
|
||||
"""Whether `t`'s elements fill a contiguous storage range, possibly
|
||||
permuted (e.g. a transposed matrix). Such layouts can be copied with
|
||||
pitched cudaMemcpy2D/3DAsync directly from/to pinned memory, whereas
|
||||
gapped layouts (e.g. strided slices) stage through a pageable temp."""
|
||||
if t.is_contiguous():
|
||||
# The common case, answered by a memoized TensorImpl flag.
|
||||
return True
|
||||
expected_stride = 1
|
||||
for stride, size in sorted(zip(t.stride(), t.shape)):
|
||||
if size <= 1:
|
||||
continue
|
||||
if stride != expected_stride:
|
||||
return False
|
||||
expected_stride *= size
|
||||
return True
|
||||
|
||||
|
||||
def _cpu_copy_stall_reason(
|
||||
cpu_tensor: torch.Tensor, device: torch.device | None
|
||||
) -> str | None:
|
||||
"""Why a `non_blocking` copy between `cpu_tensor` and a CUDA tensor would
|
||||
silently block the host, or None if it is genuinely asynchronous.
|
||||
|
||||
`device` describes the other side of the copy; a non-CUDA `device` means
|
||||
the copy does not involve the GPU at all.
|
||||
"""
|
||||
if device is None or device.type != "cuda" or cpu_tensor.device.type != "cpu":
|
||||
return None
|
||||
if cpu_tensor.numel() == 0:
|
||||
return None # Nothing to copy; torch issues no CUDA call at all.
|
||||
if not _is_dense(cpu_tensor):
|
||||
return "the CPU tensor is not densely laid out"
|
||||
if not cpu_tensor.is_pinned():
|
||||
return "the CPU tensor is not pinned"
|
||||
return None
|
||||
|
||||
|
||||
def _report_implicit_sync(op: str, direction: str, reason: str, mode: str) -> None:
|
||||
msg = (
|
||||
f"Tensor.{op}: a non_blocking {direction} copy that may be silently "
|
||||
f"synchronous ({reason}). {IMPLICIT_SYNC_ERROR_MESSAGE}"
|
||||
)
|
||||
if mode == "error":
|
||||
raise RuntimeError(msg)
|
||||
warnings.warn(msg, stacklevel=3)
|
||||
|
||||
|
||||
def _checked_to(self, *args, **kwargs):
|
||||
# Only H2D is checked: D2H `to` allocates a pinned destination in
|
||||
# supported torch versions, so it is genuinely asynchronous.
|
||||
if (mode := _active_check_mode()) is not None and _parse_to is not None:
|
||||
try:
|
||||
device, _, non_blocking, _ = _parse_to(*args, **kwargs)
|
||||
except Exception:
|
||||
pass # Invalid args; let the original call raise.
|
||||
else:
|
||||
if non_blocking and (reason := _cpu_copy_stall_reason(self, device)):
|
||||
_report_implicit_sync("to", "H2D", reason, mode)
|
||||
return _torch_to(self, *args, **kwargs)
|
||||
|
||||
|
||||
def _checked_cuda(self, *args, **kwargs):
|
||||
# Signature: cuda(device=None, non_blocking=False, memory_format=None).
|
||||
if (mode := _active_check_mode()) is not None:
|
||||
device = args[0] if args else kwargs.get("device")
|
||||
non_blocking = args[1] if len(args) > 1 else kwargs.get("non_blocking", False)
|
||||
if non_blocking:
|
||||
if device is None:
|
||||
device = torch.device("cuda")
|
||||
elif isinstance(device, int):
|
||||
device = torch.device("cuda", device)
|
||||
elif not isinstance(device, torch.device):
|
||||
device = torch.device(device)
|
||||
if reason := _cpu_copy_stall_reason(self, device):
|
||||
_report_implicit_sync("cuda", "H2D", reason, mode)
|
||||
return _torch_cuda(self, *args, **kwargs)
|
||||
|
||||
|
||||
def _checked_copy_(self, *args, **kwargs):
|
||||
# Signature: copy_(src, non_blocking=False).
|
||||
if (
|
||||
(mode := _active_check_mode()) is not None
|
||||
and args
|
||||
and isinstance(src := args[0], torch.Tensor)
|
||||
):
|
||||
non_blocking = args[1] if len(args) > 1 else kwargs.get("non_blocking", False)
|
||||
if non_blocking:
|
||||
if self.device.type == "cuda":
|
||||
reason = _cpu_copy_stall_reason(src, self.device)
|
||||
direction = "H2D"
|
||||
else:
|
||||
reason = _cpu_copy_stall_reason(self, src.device)
|
||||
direction = "D2H"
|
||||
if reason:
|
||||
_report_implicit_sync("copy_", direction, reason, mode)
|
||||
return _torch_copy_(self, *args, **kwargs)
|
||||
|
||||
|
||||
_copy_checkers_installed: bool = False
|
||||
|
||||
|
||||
def _install_copy_checkers() -> None:
|
||||
"""Wrap the Python-level CPU<->CUDA transfer entry points with the checks
|
||||
above.
|
||||
|
||||
The patch is process-global, but it only reports on threads inside a
|
||||
checked region, matching the scoping of the torch-level check. Copies
|
||||
issued from C++ (e.g. inside compiled graphs) bypass it.
|
||||
"""
|
||||
global _copy_checkers_installed
|
||||
if _copy_checkers_installed:
|
||||
return
|
||||
_copy_checkers_installed = True
|
||||
torch.Tensor.to = _checked_to # type: ignore[method-assign]
|
||||
torch.Tensor.cuda = _checked_cuda # type: ignore[method-assign]
|
||||
torch.Tensor.copy_ = _checked_copy_ # type: ignore[method-assign]
|
||||
|
||||
|
||||
def gpu_sync_allowed(first_only: bool = False):
|
||||
"""Allow GPU<->CPU syncs inside the `with` block, on this thread only.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user