From a810316c0894b8a43e53458ae5b14c9ad0b18b5d Mon Sep 17 00:00:00 2001 From: Guanxin Li <38149783+guanxingithub@users.noreply.github.com> Date: Thu, 1 Oct 2026 20:19:28 -0700 Subject: [PATCH] [Bugfix] Keep batch-invariance NCCL pins out of the weight-transfer group (#59500) Signed-off-by: aoshen524 Co-authored-by: aoshen524 Co-authored-by: Claude Opus 5.5 Co-authored-by: aoshen02 --- tests/distributed/test_weight_transfer.py | 48 ++++++++++++-- .../weight_transfer/nccl_common.py | 17 +++-- .../determinism/batch_invariant.py | 37 ++++++----- vllm/utils/nccl.py | 65 +++++++++++++++++++ 4 files changed, 140 insertions(+), 27 deletions(-) diff --git a/tests/distributed/test_weight_transfer.py b/tests/distributed/test_weight_transfer.py index 3af7b7aeb9..85ae7f58c7 100644 --- a/tests/distributed/test_weight_transfer.py +++ b/tests/distributed/test_weight_transfer.py @@ -68,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import ( SparseWeightPatch, ) from vllm.platforms import current_platform +from vllm.utils.nccl import _nccl_has_no_cache from vllm.utils.network_utils import get_open_port @@ -433,13 +434,15 @@ def trainer_broadcast_tensor( return True -@ray.remote(num_gpus=1) +# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process. +@ray.remote(num_gpus=1, max_calls=1) def inference_receive_tensor( master_address: str, master_port: int, world_size: int, tensor_shape: list[int], tensor_dtype: str, + batch_invariant: bool = False, ) -> dict: """Inference task that receives tensor via NCCLWeightTransferEngine.""" import contextlib @@ -448,6 +451,21 @@ def inference_receive_tensor( import torch _set_ray_assigned_device() + if batch_invariant: + from vllm.model_executor.determinism.batch_invariant import ( + override_envs_for_invariance, + ) + + override_envs_for_invariance() + # Like vLLM's own groups, a pinned communicator makes NCCL read the + # pins before the transfer group exists. + torch.distributed.init_process_group( + "nccl", + init_method=f"tcp://127.0.0.1:{get_open_port()}", + rank=0, + world_size=1, + ) + torch.distributed.all_reduce(torch.ones(1, device="cuda")) from vllm.config.parallel import ParallelConfig from vllm.config.weight_transfer import WeightTransferConfig @@ -534,11 +552,25 @@ def inference_receive_tensor( torch.accelerator.device_count() < 2, reason="Need at least 2 GPUs to run NCCL weight transfer test.", ) -def test_nccl_weight_transfer_between_processes(): +@pytest.mark.parametrize( + "batch_invariant", + [ + pytest.param(False, id="default"), + pytest.param( + True, + id="batch-invariant-worker", + marks=pytest.mark.skipif( + not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7." + ), + ), + ], +) +def test_nccl_weight_transfer_between_processes(batch_invariant): """Test NCCL weight transfer from trainer to inference process using Ray. This test verifies that the NCCLWeightTransferEngine can receive - tensors broadcast by a trainer process via NCCL. + tensors broadcast by a trainer process via NCCL, including when the + worker runs with batch-invariance NCCL pins the trainer does not have. """ _init_ray_for_weight_transfer() @@ -550,13 +582,19 @@ def test_nccl_weight_transfer_between_processes(): tensor_dtype = "float32" inference_future = inference_receive_tensor.remote( - master_address, master_port, world_size, tensor_shape, tensor_dtype + master_address, + master_port, + world_size, + tensor_shape, + tensor_dtype, + batch_invariant, ) trainer_future = trainer_broadcast_tensor.remote( master_address, master_port, world_size, tensor_shape, tensor_dtype ) - trainer_result, result = ray.get([trainer_future, inference_future]) + # A mismatched NCCL config deadlocks instead of failing. + trainer_result, result = ray.get([trainer_future, inference_future], timeout=300) assert trainer_result, "Trainer should complete successfully" assert result["success"], ( diff --git a/vllm/distributed/weight_transfer/nccl_common.py b/vllm/distributed/weight_transfer/nccl_common.py index a329e456d5..1cd026fb07 100644 --- a/vllm/distributed/weight_transfer/nccl_common.py +++ b/vllm/distributed/weight_transfer/nccl_common.py @@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.packed_tensor import ( DEFAULT_PACKED_BUFFER_SIZE_BYTES, DEFAULT_PACKED_NUM_BUFFERS, ) +from vllm.utils.nccl import unpinned_nccl_env def decode_nccl_unique_id( @@ -154,7 +155,8 @@ def stateless_init_process_group( pg = StatelessProcessGroup.create( host=master_address, port=master_port, rank=rank, world_size=world_size ) - return PyNcclCommunicator(pg, device=device) + with unpinned_nccl_env(): + return PyNcclCommunicator(pg, device=device) def uid_init_process_group( @@ -170,12 +172,13 @@ def uid_init_process_group( """ from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator - return PyNcclCommunicator.from_unique_id_bytes( - nccl_unique_id_bytes, - rank=rank, - world_size=world_size, - device=device, - ) + with unpinned_nccl_env(): + return PyNcclCommunicator.from_unique_id_bytes( + nccl_unique_id_bytes, + rank=rank, + world_size=world_size, + device=device, + ) def worker_init_process_group( diff --git a/vllm/model_executor/determinism/batch_invariant.py b/vllm/model_executor/determinism/batch_invariant.py index 688e14d709..6490e69a60 100644 --- a/vllm/model_executor/determinism/batch_invariant.py +++ b/vllm/model_executor/determinism/batch_invariant.py @@ -16,6 +16,7 @@ from vllm.model_executor.determinism.batch_invariant_configs import ( from vllm.platforms import current_platform from vllm.triton_utils import tl, triton from vllm.utils.mem_utils import get_max_shared_memory_bytes +from vllm.utils.nccl import pin_nccl_env from vllm.utils.platform_utils import num_compute_units from vllm.utils.torch_utils import is_torch_equal_or_newer @@ -1144,22 +1145,28 @@ def override_envs_for_invariance(): os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8" - # NCCL determinism settings + # NCCL determinism settings. NCCL keeps the launch mode process-wide, so, + # unlike the pins below, it also applies to communicators shared with other + # processes. os.environ["NCCL_LAUNCH_MODE"] = "GROUP" - os.environ["NCCL_COLLNET_ENABLE"] = "0" - os.environ["NCCL_NVLS_ENABLE"] = "0" - os.environ["NCCL_P2P_NET_DISABLE"] = "1" - os.environ["NCCL_MIN_NCHANNELS"] = "1" - os.environ["NCCL_MAX_NCHANNELS"] = "1" - os.environ["NCCL_PROTO"] = "Simple" - # NCCL >= 2.31 zero-fills the algorithm table of every collective when - # NCCL_ALGO is set and re-enables only the named ones; together with the - # NCCL_PROTO above, collectives not named here end up with no algorithm - # and fail with ncclInvalidUsage. Re-enable Ring and Tree for all - # collectives, then pin AllReduce to Tree for determinism. - os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree" - os.environ["NCCL_NTHREADS"] = "1" - os.environ["NCCL_SOCKET_NTHREADS"] = "1" + pin_nccl_env( + { + "NCCL_COLLNET_ENABLE": "0", + "NCCL_NVLS_ENABLE": "0", + "NCCL_P2P_NET_DISABLE": "1", + "NCCL_MIN_NCHANNELS": "1", + "NCCL_MAX_NCHANNELS": "1", + "NCCL_PROTO": "Simple", + # NCCL >= 2.31 zero-fills the algorithm table of every collective + # when NCCL_ALGO is set and re-enables only the named ones; together + # with the NCCL_PROTO above, collectives not named here end up with + # no algorithm and fail with ncclInvalidUsage. Re-enable Ring and + # Tree for all collectives, then pin AllReduce to Tree. + "NCCL_ALGO": "ring,tree;allreduce:tree", + "NCCL_NTHREADS": "1", + "NCCL_SOCKET_NTHREADS": "1", + } + ) # torch.compile settings os.environ["VLLM_USE_AOT_COMPILE"] = "0" diff --git a/vllm/utils/nccl.py b/vllm/utils/nccl.py index 90064d4184..1136661f6e 100644 --- a/vllm/utils/nccl.py +++ b/vllm/utils/nccl.py @@ -3,9 +3,11 @@ from __future__ import annotations +import contextlib import ctypes import importlib.util import os +from collections.abc import Iterator, Mapping import torch @@ -129,3 +131,66 @@ def query_nccl_gin_type( logger.warning("ncclCommQueryProperties returned error %d", result) return None return props.railedGinType if railed else props.ginType + + +# Values the variables set by `pin_nccl_env` had before it: None until something +# is pinned, empty if NCCL cannot re-read them. +_unpinned_env: dict[str, str | None] | None = None + + +def pin_nccl_env(pins: dict[str, str]) -> None: + """Set NCCL variables for this process's own communicators. + + NCCL never reconciles these across ranks, so communicators shared with a + process that does not set them are created without them, see + `unpinned_nccl_env`. NCCL caches most variables on first read; listing the + pins in NCCL_NO_CACHE makes it re-read them per communicator. NCCL parses + that list once, so this must run before the process's first NCCL call. + """ + global _unpinned_env + launch_env = _swap_env(pins) + if _unpinned_env is not None: + return + _unpinned_env = launch_env if _nccl_has_no_cache() else {} + if _unpinned_env: + no_cache = os.environ.get("NCCL_NO_CACHE") + os.environ["NCCL_NO_CACHE"] = ",".join(filter(None, [no_cache, *pins])) + + +@contextlib.contextmanager +def unpinned_nccl_env() -> Iterator[None]: + """Create communicators shared with other processes without the pins.""" + if _unpinned_env == {}: + logger.warning_once( + "NCCL_NO_CACHE needs CUDA NCCL >= 2.29.7, so the NCCL settings vLLM " + "pinned also apply here; a process sharing this communicator must " + "set the same NCCL_* variables or it will hang." + ) + pinned = _swap_env(_unpinned_env or {}) + try: + yield + finally: + _swap_env(pinned) + + +def _nccl_has_no_cache() -> bool: + # NCCL_NO_CACHE arrived in NCCL 2.29.7; ask the library vLLM loads. + if torch.version.cuda is None: + return False + version = ctypes.c_int() + try: + ctypes.CDLL(find_nccl_library()).ncclGetVersion(ctypes.byref(version)) + except Exception: + return False + return version.value >= 22907 + + +def _swap_env(values: Mapping[str, str | None]) -> dict[str, str | None]: + """Set (None: unset) the given variables and return their previous values.""" + old = {name: os.environ.get(name) for name in values} + for name, value in values.items(): + if value is None: + os.environ.pop(name, None) + else: + os.environ[name] = value + return old