diff --git a/tests/utils_/test_gpu_sync_debug.py b/tests/utils_/test_gpu_sync_debug.py index 12001a026a..dc8b122800 100644 --- a/tests/utils_/test_gpu_sync_debug.py +++ b/tests/utils_/test_gpu_sync_debug.py @@ -158,3 +158,149 @@ def test_without_env_set(monkeypatch): with_gpu_sync_check(_no_sync)() with_gpu_sync_check(_causes_sync)() + + +def _pageable_h2d_nonblocking(): + # Pageable source: the "async" copy is staged through pageable memory. + torch.zeros(4).to("cuda", non_blocking=True) + + +def _pinned_h2d_nonblocking(): + # Pinned, contiguous, same dtype: genuinely asynchronous. + torch.zeros(4).pin_memory().to("cuda", non_blocking=True) + + +def _noncontiguous_h2d_nonblocking(): + # Pinned but gapped layout (strided slice): staged through a pageable temp. + torch.zeros(8).pin_memory()[::2].to("cuda", non_blocking=True) + + +def _transposed_h2d_nonblocking(): + # Dense permuted layout: copied with pitched cudaMemcpy2DAsync, stays async. + torch.zeros(4, 4).pin_memory().t().to("cuda", non_blocking=True) + + +def _dtype_converting_h2d_nonblocking(): + # Pinned and contiguous: the conversion runs GPU-side, staying async. + torch.zeros(4, dtype=torch.float64).pin_memory().to( + "cuda", dtype=torch.float32, non_blocking=True + ) + + +def _pageable_h2d_via_cuda(): + torch.zeros(4).cuda(non_blocking=True) + + +def _pageable_h2d_via_copy_(): + torch.zeros(4, device="cuda").copy_(torch.zeros(4), non_blocking=True) + + +def _d2h_via_to(): + # Genuinely asynchronous: torch allocates the destination pinned. + torch.zeros(4, device="cuda").to("cpu", non_blocking=True) + + +def _pageable_d2h_via_copy_(): + torch.zeros(4).copy_(torch.zeros(4, device="cuda"), non_blocking=True) + + +def _pinned_d2h_via_copy_(): + # Pinned, contiguous, same dtype: genuinely asynchronous. + torch.zeros(4).pin_memory().copy_(torch.zeros(4, device="cuda"), non_blocking=True) + + +def _noncontiguous_d2h_via_copy_(): + # Pinned but gapped (strided slice) destination. + torch.zeros(8).pin_memory()[::2].copy_( + torch.zeros(4, device="cuda"), non_blocking=True + ) + + +def _transposed_d2h_via_copy_(): + # Dense permuted destination: pitched cudaMemcpy2DAsync, stays async. + torch.zeros(4, 4).pin_memory().t().copy_( + torch.zeros(4, 4, device="cuda"), non_blocking=True + ) + + +def _empty_h2d_nonblocking(): + # Empty (e.g. first-step penalties): no CUDA call is issued at all. + torch.zeros(0).to("cuda", non_blocking=True) + + +def _dtype_converting_d2h_via_copy_(): + # Pinned and contiguous: the conversion runs GPU-side, staying async. + torch.zeros(4).pin_memory().copy_( + torch.zeros(4, dtype=torch.float64, device="cuda"), non_blocking=True + ) + + +@pytest.mark.parametrize("mode", ["warn", "error"]) +@pytest.mark.parametrize( + "fn", + [ + _pageable_h2d_nonblocking, + _noncontiguous_h2d_nonblocking, + _pageable_h2d_via_cuda, + _pageable_h2d_via_copy_, + _pageable_d2h_via_copy_, + _noncontiguous_d2h_via_copy_, + ], +) +@create_new_process_for_each_test() +def test_implicit_copy_sync_detected(monkeypatch, mode, fn): + """`non_blocking=True` CPU<->CUDA copies with a pageable or + non-densely-laid-out CPU tensor may block the host without tripping + torch's sync debug mode; the `Tensor.to`/`cuda`/`copy_` wrappers must + flag them. + """ + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", mode) + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + gsd._install_copy_checkers() + + if mode == "error": + with pytest.raises(RuntimeError, match="Implicit GPU<->CPU sync"): + with_gpu_sync_check(fn)() + else: + with pytest.warns(UserWarning, match="Implicit GPU<->CPU sync"): + with_gpu_sync_check(fn)() + + +@create_new_process_for_each_test() +def test_genuinely_async_transfers_pass(monkeypatch): + """Pinned, densely laid out CPU tensors make `non_blocking=True` truly + asynchronous in both directions -- even with a dtype conversion, which + runs GPU-side, and even permuted (e.g. transposed), which uses pitched + cudaMemcpy2D/3DAsync; D2H `Tensor.to` allocates a pinned destination. + None of these may be flagged.""" + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + gsd._install_copy_checkers() + + for fn in ( + _pinned_h2d_nonblocking, + _pinned_d2h_via_copy_, + _d2h_via_to, + _dtype_converting_h2d_nonblocking, + _dtype_converting_d2h_via_copy_, + _transposed_h2d_nonblocking, + _transposed_d2h_via_copy_, + _empty_h2d_nonblocking, + ): + fn() # Warm up outside the checked region. + with_gpu_sync_check(fn)() + + +@create_new_process_for_each_test() +def test_implicit_copy_sync_can_be_allowed(monkeypatch): + """`gpu_sync_allowed()` must exempt implicit copy syncs too.""" + monkeypatch.setattr(gsd, "_SYNC_CHECK_MODE", "error") + monkeypatch.setattr(gsd, "_sync_check_enabled", True) + gsd._install_copy_checkers() + + def allowed_pageable_copies(): + with gpu_sync_allowed(): + _pageable_h2d_nonblocking() + _pageable_d2h_via_copy_() + + with_gpu_sync_check(allowed_pageable_copies)() diff --git a/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py b/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py index 3517d55ecd..750d3b56d9 100644 --- a/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py +++ b/vllm/distributed/kv_transfer/kv_connector/v1/example_connector.py @@ -135,7 +135,11 @@ class ExampleConnector(KVConnectorBase_V1): slot_mapping (torch.Tensor): the slot mapping. In shape [num_tokens]. """ - slot_mapping = slot_mapping.to(dst_kv_cache_layer.device, non_blocking=True) + # Deliberate bulk transfers in this reference connector. + with gpu_sync_allowed(): + slot_mapping = slot_mapping.to( + dst_kv_cache_layer.device, non_blocking=True + ) if isinstance(attn_metadata, MLACommonMetadata): # [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots. dst_kv_cache_layer = dst_kv_cache_layer.reshape( @@ -177,8 +181,10 @@ class ExampleConnector(KVConnectorBase_V1): filename = self._generate_filename_debug( layer_name, request.token_ids, request.mm_hashes ) - kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"] - kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True) + # Deliberate bulk load in this reference connector. + with gpu_sync_allowed(): + kv_cache_cpu = safetensors.torch.load_file(filename)["kv_cache"] + kv_cache = kv_cache_cpu.to(kv_cache_layer.device, non_blocking=True) if isinstance(attn_metadata, dict): inject_kv_into_layer( kv_cache_layer, @@ -224,7 +230,9 @@ class ExampleConnector(KVConnectorBase_V1): The layer is a standardized [B, H, N, C] per-layer view (H == 1 for MLA). """ - slot_mapping = slot_mapping.to(layer.device, non_blocking=True) + # Deliberate bulk transfer in this reference connector. + with gpu_sync_allowed(): + slot_mapping = slot_mapping.to(layer.device, non_blocking=True) if isinstance(attn_metadata, MLACommonMetadata): # [B, 1, N, C] -> [B * N, C]; slot_mapping indexes B * N slots. return layer.reshape(-1, layer.shape[-1])[slot_mapping, ...] diff --git a/vllm/envs.py b/vllm/envs.py index a3ef86d1b6..ae506acee2 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -642,9 +642,11 @@ environment_variables: dict[str, Callable[[], Any]] = { "VLLM_TRITON_USE_TD": lambda: {"1": True, "0": False}.get( os.getenv("VLLM_TRITON_USE_TD", "").strip() ), - # If set, enable PyTorch's GPU<->CPU synchronization debug mode around - # the worker's `execute_model` and `sample_tokens` calls. Valid values - # are "warn" (print a warning on each sync) or "error" (raise on sync). + # If set, enable GPU<->CPU synchronization checking around the worker's + # `execute_model` and `sample_tokens` calls, via PyTorch's sync debug mode + # plus wrappers flagging `non_blocking` CPU<->CUDA copies that silently + # block the host (CPU tensors that are not pinned or not densely laid out). + # Valid values are "warn" (warn on each sync) or "error" (raise on sync). # Unset disables the check. See `torch.cuda.set_sync_debug_mode`. "VLLM_GPU_SYNC_CHECK": env_with_choices( "VLLM_GPU_SYNC_CHECK", None, ["warn", "error"] diff --git a/vllm/model_executor/offloader/uva.py b/vllm/model_executor/offloader/uva.py index 5107870ffa..f04d4d85b1 100644 --- a/vllm/model_executor/offloader/uva.py +++ b/vllm/model_executor/offloader/uva.py @@ -3,6 +3,7 @@ """UVA-based CPU offloading using Unified Virtual Addressing.""" from collections.abc import Generator +from contextlib import nullcontext import torch import torch.nn as nn @@ -11,6 +12,7 @@ from torch.func import functional_call import vllm.envs as envs from vllm.logger import init_logger from vllm.model_executor.offloader.base import BaseOffloader, should_pin_memory +from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.mem_utils import format_gib from vllm.utils.platform_utils import is_uva_available from vllm.utils.torch_utils import get_accelerator_view_from_cpu_tensor @@ -126,13 +128,14 @@ class UVAOffloader(BaseOffloader): def forward(*args, **kwargs): module.forward = original_forward - device_state = { - # here we blindly call `to(device)` - # if the parameter is already on the device, - # it will be a no-op - k: v.to(device, non_blocking=True) - for k, v in module.state_dict().items() - } + with nullcontext() if self.pin_memory else gpu_sync_allowed(): + device_state = { + # here we blindly call `to(device)` + # if the parameter is already on the device, + # it will be a no-op + k: v.to(device, non_blocking=True) + for k, v in module.state_dict().items() + } # set `tie_weights=False` as tied weights in original model # become untied when calling .to(device) individually diff --git a/vllm/utils/gpu_sync_debug.py b/vllm/utils/gpu_sync_debug.py index 758b73e842..53fcdfe710 100644 --- a/vllm/utils/gpu_sync_debug.py +++ b/vllm/utils/gpu_sync_debug.py @@ -7,6 +7,12 @@ is a failure. The scoping lives in ContextVars, which are per-thread and per-asyncio-task, so an allow region opened on one thread is invisible to every other one. + +torch's mode only instruments explicit synchronizing calls, so it misses +`non_blocking=True` CPU<->CUDA copies that the CUDA driver silently stages +through pageable memory (when the CPU tensor is not pinned or not densely +laid out). Those are caught by the wrappers on `Tensor.to`/`Tensor.cuda`/ +`Tensor.copy_` installed by `_install_copy_checkers()`. """ import functools @@ -54,6 +60,7 @@ def enable_gpu_sync_check() -> None: global _sync_check_enabled _sync_check_enabled = True _install_compile_time_sync_suppressors() + _install_copy_checkers() _arm_lock = threading.Lock() @@ -197,6 +204,158 @@ def _install_compile_time_sync_suppressors() -> None: pass +# --------------------------------------------------------------------------- +# Implicit sync detection in `non_blocking` CPU<->CUDA copies. +# +# A `non_blocking=True` copy between CPU and CUDA still blocks the host when +# the CPU side is not pinned or not densely laid out (gapped strided slices +# stage through a pageable temp): D2H copies into such memory wait for the +# stream, and H2D copies from it do too beyond a driver size threshold. +# None of that goes through an API torch's sync debug mode instruments, so +# wrap the Python-level transfer entry points and flag such copies on threads +# in a checked region. + +IMPLICIT_SYNC_ERROR_MESSAGE = ( + "Implicit GPU<->CPU sync detected - use a pinned, densely laid out CPU " + "tensor, or wrap with gpu_sync_allowed()" +) + +# Captured at import; `_install_copy_checkers` replaces these attributes. +_torch_to = torch.Tensor.to +_torch_cuda = torch.Tensor.cuda +_torch_copy_ = torch.Tensor.copy_ + +# torch's own parser for `Tensor.to`'s overloaded signature, returning +# (device, dtype, non_blocking, memory_format) (also used by torch._dynamo). +_parse_to = getattr(torch._C._nn, "_parse_to", None) + + +def _active_check_mode() -> str | None: + """The mode the calling thread is being checked in, if any.""" + if not _sync_check_enabled or _allow_depth.get() or torch.compiler.is_compiling(): + return None + return _checking.get() + + +def _is_dense(t: torch.Tensor) -> bool: + """Whether `t`'s elements fill a contiguous storage range, possibly + permuted (e.g. a transposed matrix). Such layouts can be copied with + pitched cudaMemcpy2D/3DAsync directly from/to pinned memory, whereas + gapped layouts (e.g. strided slices) stage through a pageable temp.""" + if t.is_contiguous(): + # The common case, answered by a memoized TensorImpl flag. + return True + expected_stride = 1 + for stride, size in sorted(zip(t.stride(), t.shape)): + if size <= 1: + continue + if stride != expected_stride: + return False + expected_stride *= size + return True + + +def _cpu_copy_stall_reason( + cpu_tensor: torch.Tensor, device: torch.device | None +) -> str | None: + """Why a `non_blocking` copy between `cpu_tensor` and a CUDA tensor would + silently block the host, or None if it is genuinely asynchronous. + + `device` describes the other side of the copy; a non-CUDA `device` means + the copy does not involve the GPU at all. + """ + if device is None or device.type != "cuda" or cpu_tensor.device.type != "cpu": + return None + if cpu_tensor.numel() == 0: + return None # Nothing to copy; torch issues no CUDA call at all. + if not _is_dense(cpu_tensor): + return "the CPU tensor is not densely laid out" + if not cpu_tensor.is_pinned(): + return "the CPU tensor is not pinned" + return None + + +def _report_implicit_sync(op: str, direction: str, reason: str, mode: str) -> None: + msg = ( + f"Tensor.{op}: a non_blocking {direction} copy that may be silently " + f"synchronous ({reason}). {IMPLICIT_SYNC_ERROR_MESSAGE}" + ) + if mode == "error": + raise RuntimeError(msg) + warnings.warn(msg, stacklevel=3) + + +def _checked_to(self, *args, **kwargs): + # Only H2D is checked: D2H `to` allocates a pinned destination in + # supported torch versions, so it is genuinely asynchronous. + if (mode := _active_check_mode()) is not None and _parse_to is not None: + try: + device, _, non_blocking, _ = _parse_to(*args, **kwargs) + except Exception: + pass # Invalid args; let the original call raise. + else: + if non_blocking and (reason := _cpu_copy_stall_reason(self, device)): + _report_implicit_sync("to", "H2D", reason, mode) + return _torch_to(self, *args, **kwargs) + + +def _checked_cuda(self, *args, **kwargs): + # Signature: cuda(device=None, non_blocking=False, memory_format=None). + if (mode := _active_check_mode()) is not None: + device = args[0] if args else kwargs.get("device") + non_blocking = args[1] if len(args) > 1 else kwargs.get("non_blocking", False) + if non_blocking: + if device is None: + device = torch.device("cuda") + elif isinstance(device, int): + device = torch.device("cuda", device) + elif not isinstance(device, torch.device): + device = torch.device(device) + if reason := _cpu_copy_stall_reason(self, device): + _report_implicit_sync("cuda", "H2D", reason, mode) + return _torch_cuda(self, *args, **kwargs) + + +def _checked_copy_(self, *args, **kwargs): + # Signature: copy_(src, non_blocking=False). + if ( + (mode := _active_check_mode()) is not None + and args + and isinstance(src := args[0], torch.Tensor) + ): + non_blocking = args[1] if len(args) > 1 else kwargs.get("non_blocking", False) + if non_blocking: + if self.device.type == "cuda": + reason = _cpu_copy_stall_reason(src, self.device) + direction = "H2D" + else: + reason = _cpu_copy_stall_reason(self, src.device) + direction = "D2H" + if reason: + _report_implicit_sync("copy_", direction, reason, mode) + return _torch_copy_(self, *args, **kwargs) + + +_copy_checkers_installed: bool = False + + +def _install_copy_checkers() -> None: + """Wrap the Python-level CPU<->CUDA transfer entry points with the checks + above. + + The patch is process-global, but it only reports on threads inside a + checked region, matching the scoping of the torch-level check. Copies + issued from C++ (e.g. inside compiled graphs) bypass it. + """ + global _copy_checkers_installed + if _copy_checkers_installed: + return + _copy_checkers_installed = True + torch.Tensor.to = _checked_to # type: ignore[method-assign] + torch.Tensor.cuda = _checked_cuda # type: ignore[method-assign] + torch.Tensor.copy_ = _checked_copy_ # type: ignore[method-assign] + + def gpu_sync_allowed(first_only: bool = False): """Allow GPU<->CPU syncs inside the `with` block, on this thread only.