[Bugfix] Keep batch-invariance NCCL pins out of the weight-transfer group (#59500)

Signed-off-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Guanxin Li
2026-10-02 11:19:28 +08:00
committed by GitHub
co-authored by aoshen524 Claude Opus 5.5 aoshen02
parent fcfacc3499
commit a810316c08
4 changed files with 140 additions and 27 deletions
+65
View File
@@ -3,9 +3,11 @@
from __future__ import annotations
import contextlib
import ctypes
import importlib.util
import os
from collections.abc import Iterator, Mapping
import torch
@@ -129,3 +131,66 @@ def query_nccl_gin_type(
logger.warning("ncclCommQueryProperties returned error %d", result)
return None
return props.railedGinType if railed else props.ginType
# Values the variables set by `pin_nccl_env` had before it: None until something
# is pinned, empty if NCCL cannot re-read them.
_unpinned_env: dict[str, str | None] | None = None
def pin_nccl_env(pins: dict[str, str]) -> None:
"""Set NCCL variables for this process's own communicators.
NCCL never reconciles these across ranks, so communicators shared with a
process that does not set them are created without them, see
`unpinned_nccl_env`. NCCL caches most variables on first read; listing the
pins in NCCL_NO_CACHE makes it re-read them per communicator. NCCL parses
that list once, so this must run before the process's first NCCL call.
"""
global _unpinned_env
launch_env = _swap_env(pins)
if _unpinned_env is not None:
return
_unpinned_env = launch_env if _nccl_has_no_cache() else {}
if _unpinned_env:
no_cache = os.environ.get("NCCL_NO_CACHE")
os.environ["NCCL_NO_CACHE"] = ",".join(filter(None, [no_cache, *pins]))
@contextlib.contextmanager
def unpinned_nccl_env() -> Iterator[None]:
"""Create communicators shared with other processes without the pins."""
if _unpinned_env == {}:
logger.warning_once(
"NCCL_NO_CACHE needs CUDA NCCL >= 2.29.7, so the NCCL settings vLLM "
"pinned also apply here; a process sharing this communicator must "
"set the same NCCL_* variables or it will hang."
)
pinned = _swap_env(_unpinned_env or {})
try:
yield
finally:
_swap_env(pinned)
def _nccl_has_no_cache() -> bool:
# NCCL_NO_CACHE arrived in NCCL 2.29.7; ask the library vLLM loads.
if torch.version.cuda is None:
return False
version = ctypes.c_int()
try:
ctypes.CDLL(find_nccl_library()).ncclGetVersion(ctypes.byref(version))
except Exception:
return False
return version.value >= 22907
def _swap_env(values: Mapping[str, str | None]) -> dict[str, str | None]:
"""Set (None: unset) the given variables and return their previous values."""
old = {name: os.environ.get(name) for name in values}
for name, value in values.items():
if value is None:
os.environ.pop(name, None)
else:
os.environ[name] = value
return old