[Bugfix] Keep batch-invariance NCCL pins out of the weight-transfer group (#59500)

Signed-off-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Guanxin Li
2026-10-02 11:19:28 +08:00
committed by GitHub
co-authored by aoshen524 Claude Opus 5.5 aoshen02
parent fcfacc3499
commit a810316c08
4 changed files with 140 additions and 27 deletions
+43 -5
View File
@@ -68,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseWeightPatch,
)
from vllm.platforms import current_platform
from vllm.utils.nccl import _nccl_has_no_cache
from vllm.utils.network_utils import get_open_port
@@ -433,13 +434,15 @@ def trainer_broadcast_tensor(
return True
@ray.remote(num_gpus=1)
# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process.
@ray.remote(num_gpus=1, max_calls=1)
def inference_receive_tensor(
master_address: str,
master_port: int,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
batch_invariant: bool = False,
) -> dict:
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
import contextlib
@@ -448,6 +451,21 @@ def inference_receive_tensor(
import torch
_set_ray_assigned_device()
if batch_invariant:
from vllm.model_executor.determinism.batch_invariant import (
override_envs_for_invariance,
)
override_envs_for_invariance()
# Like vLLM's own groups, a pinned communicator makes NCCL read the
# pins before the transfer group exists.
torch.distributed.init_process_group(
"nccl",
init_method=f"tcp://127.0.0.1:{get_open_port()}",
rank=0,
world_size=1,
)
torch.distributed.all_reduce(torch.ones(1, device="cuda"))
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
@@ -534,11 +552,25 @@ def inference_receive_tensor(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_between_processes():
@pytest.mark.parametrize(
"batch_invariant",
[
pytest.param(False, id="default"),
pytest.param(
True,
id="batch-invariant-worker",
marks=pytest.mark.skipif(
not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7."
),
),
],
)
def test_nccl_weight_transfer_between_processes(batch_invariant):
"""Test NCCL weight transfer from trainer to inference process using Ray.
This test verifies that the NCCLWeightTransferEngine can receive
tensors broadcast by a trainer process via NCCL.
tensors broadcast by a trainer process via NCCL, including when the
worker runs with batch-invariance NCCL pins the trainer does not have.
"""
_init_ray_for_weight_transfer()
@@ -550,13 +582,19 @@ def test_nccl_weight_transfer_between_processes():
tensor_dtype = "float32"
inference_future = inference_receive_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
master_address,
master_port,
world_size,
tensor_shape,
tensor_dtype,
batch_invariant,
)
trainer_future = trainer_broadcast_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
trainer_result, result = ray.get([trainer_future, inference_future])
# A mismatched NCCL config deadlocks instead of failing.
trainer_result, result = ray.get([trainer_future, inference_future], timeout=300)
assert trainer_result, "Trainer should complete successfully"
assert result["success"], (
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
)
from vllm.utils.nccl import unpinned_nccl_env
def decode_nccl_unique_id(
@@ -154,7 +155,8 @@ def stateless_init_process_group(
pg = StatelessProcessGroup.create(
host=master_address, port=master_port, rank=rank, world_size=world_size
)
return PyNcclCommunicator(pg, device=device)
with unpinned_nccl_env():
return PyNcclCommunicator(pg, device=device)
def uid_init_process_group(
@@ -170,12 +172,13 @@ def uid_init_process_group(
"""
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
with unpinned_nccl_env():
return PyNcclCommunicator.from_unique_id_bytes(
nccl_unique_id_bytes,
rank=rank,
world_size=world_size,
device=device,
)
def worker_init_process_group(
@@ -16,6 +16,7 @@ from vllm.model_executor.determinism.batch_invariant_configs import (
from vllm.platforms import current_platform
from vllm.triton_utils import tl, triton
from vllm.utils.mem_utils import get_max_shared_memory_bytes
from vllm.utils.nccl import pin_nccl_env
from vllm.utils.platform_utils import num_compute_units
from vllm.utils.torch_utils import is_torch_equal_or_newer
@@ -1144,22 +1145,28 @@ def override_envs_for_invariance():
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
# NCCL determinism settings
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
# unlike the pins below, it also applies to communicators shared with other
# processes.
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
os.environ["NCCL_COLLNET_ENABLE"] = "0"
os.environ["NCCL_NVLS_ENABLE"] = "0"
os.environ["NCCL_P2P_NET_DISABLE"] = "1"
os.environ["NCCL_MIN_NCHANNELS"] = "1"
os.environ["NCCL_MAX_NCHANNELS"] = "1"
os.environ["NCCL_PROTO"] = "Simple"
# NCCL >= 2.31 zero-fills the algorithm table of every collective when
# NCCL_ALGO is set and re-enables only the named ones; together with the
# NCCL_PROTO above, collectives not named here end up with no algorithm
# and fail with ncclInvalidUsage. Re-enable Ring and Tree for all
# collectives, then pin AllReduce to Tree for determinism.
os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree"
os.environ["NCCL_NTHREADS"] = "1"
os.environ["NCCL_SOCKET_NTHREADS"] = "1"
pin_nccl_env(
{
"NCCL_COLLNET_ENABLE": "0",
"NCCL_NVLS_ENABLE": "0",
"NCCL_P2P_NET_DISABLE": "1",
"NCCL_MIN_NCHANNELS": "1",
"NCCL_MAX_NCHANNELS": "1",
"NCCL_PROTO": "Simple",
# NCCL >= 2.31 zero-fills the algorithm table of every collective
# when NCCL_ALGO is set and re-enables only the named ones; together
# with the NCCL_PROTO above, collectives not named here end up with
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
# Tree for all collectives, then pin AllReduce to Tree.
"NCCL_ALGO": "ring,tree;allreduce:tree",
"NCCL_NTHREADS": "1",
"NCCL_SOCKET_NTHREADS": "1",
}
)
# torch.compile settings
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
+65
View File
@@ -3,9 +3,11 @@
from __future__ import annotations
import contextlib
import ctypes
import importlib.util
import os
from collections.abc import Iterator, Mapping
import torch
@@ -129,3 +131,66 @@ def query_nccl_gin_type(
logger.warning("ncclCommQueryProperties returned error %d", result)
return None
return props.railedGinType if railed else props.ginType
# Values the variables set by `pin_nccl_env` had before it: None until something
# is pinned, empty if NCCL cannot re-read them.
_unpinned_env: dict[str, str | None] | None = None
def pin_nccl_env(pins: dict[str, str]) -> None:
"""Set NCCL variables for this process's own communicators.
NCCL never reconciles these across ranks, so communicators shared with a
process that does not set them are created without them, see
`unpinned_nccl_env`. NCCL caches most variables on first read; listing the
pins in NCCL_NO_CACHE makes it re-read them per communicator. NCCL parses
that list once, so this must run before the process's first NCCL call.
"""
global _unpinned_env
launch_env = _swap_env(pins)
if _unpinned_env is not None:
return
_unpinned_env = launch_env if _nccl_has_no_cache() else {}
if _unpinned_env:
no_cache = os.environ.get("NCCL_NO_CACHE")
os.environ["NCCL_NO_CACHE"] = ",".join(filter(None, [no_cache, *pins]))
@contextlib.contextmanager
def unpinned_nccl_env() -> Iterator[None]:
"""Create communicators shared with other processes without the pins."""
if _unpinned_env == {}:
logger.warning_once(
"NCCL_NO_CACHE needs CUDA NCCL >= 2.29.7, so the NCCL settings vLLM "
"pinned also apply here; a process sharing this communicator must "
"set the same NCCL_* variables or it will hang."
)
pinned = _swap_env(_unpinned_env or {})
try:
yield
finally:
_swap_env(pinned)
def _nccl_has_no_cache() -> bool:
# NCCL_NO_CACHE arrived in NCCL 2.29.7; ask the library vLLM loads.
if torch.version.cuda is None:
return False
version = ctypes.c_int()
try:
ctypes.CDLL(find_nccl_library()).ncclGetVersion(ctypes.byref(version))
except Exception:
return False
return version.value >= 22907
def _swap_env(values: Mapping[str, str | None]) -> dict[str, str | None]:
"""Set (None: unset) the given variables and return their previous values."""
old = {name: os.environ.get(name) for name in values}
for name, value in values.items():
if value is None:
os.environ.pop(name, None)
else:
os.environ[name] = value
return old