[Bugfix] Keep batch-invariance NCCL pins out of the weight-transfer group (#59500)
Signed-off-by: aoshen524 <aoshen524@gmail.com> Co-authored-by: aoshen524 <aoshen524@gmail.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
co-authored by
aoshen524
Claude Opus 5.5
aoshen02
parent
fcfacc3499
commit
a810316c08
@@ -68,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseWeightPatch,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.nccl import _nccl_has_no_cache
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
|
||||
@@ -433,13 +434,15 @@ def trainer_broadcast_tensor(
|
||||
return True
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process.
|
||||
@ray.remote(num_gpus=1, max_calls=1)
|
||||
def inference_receive_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
batch_invariant: bool = False,
|
||||
) -> dict:
|
||||
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
|
||||
import contextlib
|
||||
@@ -448,6 +451,21 @@ def inference_receive_tensor(
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
if batch_invariant:
|
||||
from vllm.model_executor.determinism.batch_invariant import (
|
||||
override_envs_for_invariance,
|
||||
)
|
||||
|
||||
override_envs_for_invariance()
|
||||
# Like vLLM's own groups, a pinned communicator makes NCCL read the
|
||||
# pins before the transfer group exists.
|
||||
torch.distributed.init_process_group(
|
||||
"nccl",
|
||||
init_method=f"tcp://127.0.0.1:{get_open_port()}",
|
||||
rank=0,
|
||||
world_size=1,
|
||||
)
|
||||
torch.distributed.all_reduce(torch.ones(1, device="cuda"))
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
@@ -534,11 +552,25 @@ def inference_receive_tensor(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes():
|
||||
@pytest.mark.parametrize(
|
||||
"batch_invariant",
|
||||
[
|
||||
pytest.param(False, id="default"),
|
||||
pytest.param(
|
||||
True,
|
||||
id="batch-invariant-worker",
|
||||
marks=pytest.mark.skipif(
|
||||
not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7."
|
||||
),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes(batch_invariant):
|
||||
"""Test NCCL weight transfer from trainer to inference process using Ray.
|
||||
|
||||
This test verifies that the NCCLWeightTransferEngine can receive
|
||||
tensors broadcast by a trainer process via NCCL.
|
||||
tensors broadcast by a trainer process via NCCL, including when the
|
||||
worker runs with batch-invariance NCCL pins the trainer does not have.
|
||||
"""
|
||||
_init_ray_for_weight_transfer()
|
||||
|
||||
@@ -550,13 +582,19 @@ def test_nccl_weight_transfer_between_processes():
|
||||
tensor_dtype = "float32"
|
||||
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
master_address,
|
||||
master_port,
|
||||
world_size,
|
||||
tensor_shape,
|
||||
tensor_dtype,
|
||||
batch_invariant,
|
||||
)
|
||||
trainer_future = trainer_broadcast_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
trainer_result, result = ray.get([trainer_future, inference_future])
|
||||
# A mismatched NCCL config deadlocks instead of failing.
|
||||
trainer_result, result = ray.get([trainer_future, inference_future], timeout=300)
|
||||
|
||||
assert trainer_result, "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
|
||||
@@ -26,6 +26,7 @@ from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
)
|
||||
from vllm.utils.nccl import unpinned_nccl_env
|
||||
|
||||
|
||||
def decode_nccl_unique_id(
|
||||
@@ -154,7 +155,8 @@ def stateless_init_process_group(
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address, port=master_port, rank=rank, world_size=world_size
|
||||
)
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
with unpinned_nccl_env():
|
||||
return PyNcclCommunicator(pg, device=device)
|
||||
|
||||
|
||||
def uid_init_process_group(
|
||||
@@ -170,12 +172,13 @@ def uid_init_process_group(
|
||||
"""
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
return PyNcclCommunicator.from_unique_id_bytes(
|
||||
nccl_unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=device,
|
||||
)
|
||||
with unpinned_nccl_env():
|
||||
return PyNcclCommunicator.from_unique_id_bytes(
|
||||
nccl_unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=device,
|
||||
)
|
||||
|
||||
|
||||
def worker_init_process_group(
|
||||
|
||||
@@ -16,6 +16,7 @@ from vllm.model_executor.determinism.batch_invariant_configs import (
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.triton_utils import tl, triton
|
||||
from vllm.utils.mem_utils import get_max_shared_memory_bytes
|
||||
from vllm.utils.nccl import pin_nccl_env
|
||||
from vllm.utils.platform_utils import num_compute_units
|
||||
from vllm.utils.torch_utils import is_torch_equal_or_newer
|
||||
|
||||
@@ -1144,22 +1145,28 @@ def override_envs_for_invariance():
|
||||
|
||||
os.environ["CUBLAS_WORKSPACE_CONFIG"] = ":4096:8"
|
||||
|
||||
# NCCL determinism settings
|
||||
# NCCL determinism settings. NCCL keeps the launch mode process-wide, so,
|
||||
# unlike the pins below, it also applies to communicators shared with other
|
||||
# processes.
|
||||
os.environ["NCCL_LAUNCH_MODE"] = "GROUP"
|
||||
os.environ["NCCL_COLLNET_ENABLE"] = "0"
|
||||
os.environ["NCCL_NVLS_ENABLE"] = "0"
|
||||
os.environ["NCCL_P2P_NET_DISABLE"] = "1"
|
||||
os.environ["NCCL_MIN_NCHANNELS"] = "1"
|
||||
os.environ["NCCL_MAX_NCHANNELS"] = "1"
|
||||
os.environ["NCCL_PROTO"] = "Simple"
|
||||
# NCCL >= 2.31 zero-fills the algorithm table of every collective when
|
||||
# NCCL_ALGO is set and re-enables only the named ones; together with the
|
||||
# NCCL_PROTO above, collectives not named here end up with no algorithm
|
||||
# and fail with ncclInvalidUsage. Re-enable Ring and Tree for all
|
||||
# collectives, then pin AllReduce to Tree for determinism.
|
||||
os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree"
|
||||
os.environ["NCCL_NTHREADS"] = "1"
|
||||
os.environ["NCCL_SOCKET_NTHREADS"] = "1"
|
||||
pin_nccl_env(
|
||||
{
|
||||
"NCCL_COLLNET_ENABLE": "0",
|
||||
"NCCL_NVLS_ENABLE": "0",
|
||||
"NCCL_P2P_NET_DISABLE": "1",
|
||||
"NCCL_MIN_NCHANNELS": "1",
|
||||
"NCCL_MAX_NCHANNELS": "1",
|
||||
"NCCL_PROTO": "Simple",
|
||||
# NCCL >= 2.31 zero-fills the algorithm table of every collective
|
||||
# when NCCL_ALGO is set and re-enables only the named ones; together
|
||||
# with the NCCL_PROTO above, collectives not named here end up with
|
||||
# no algorithm and fail with ncclInvalidUsage. Re-enable Ring and
|
||||
# Tree for all collectives, then pin AllReduce to Tree.
|
||||
"NCCL_ALGO": "ring,tree;allreduce:tree",
|
||||
"NCCL_NTHREADS": "1",
|
||||
"NCCL_SOCKET_NTHREADS": "1",
|
||||
}
|
||||
)
|
||||
|
||||
# torch.compile settings
|
||||
os.environ["VLLM_USE_AOT_COMPILE"] = "0"
|
||||
|
||||
@@ -3,9 +3,11 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import ctypes
|
||||
import importlib.util
|
||||
import os
|
||||
from collections.abc import Iterator, Mapping
|
||||
|
||||
import torch
|
||||
|
||||
@@ -129,3 +131,66 @@ def query_nccl_gin_type(
|
||||
logger.warning("ncclCommQueryProperties returned error %d", result)
|
||||
return None
|
||||
return props.railedGinType if railed else props.ginType
|
||||
|
||||
|
||||
# Values the variables set by `pin_nccl_env` had before it: None until something
|
||||
# is pinned, empty if NCCL cannot re-read them.
|
||||
_unpinned_env: dict[str, str | None] | None = None
|
||||
|
||||
|
||||
def pin_nccl_env(pins: dict[str, str]) -> None:
|
||||
"""Set NCCL variables for this process's own communicators.
|
||||
|
||||
NCCL never reconciles these across ranks, so communicators shared with a
|
||||
process that does not set them are created without them, see
|
||||
`unpinned_nccl_env`. NCCL caches most variables on first read; listing the
|
||||
pins in NCCL_NO_CACHE makes it re-read them per communicator. NCCL parses
|
||||
that list once, so this must run before the process's first NCCL call.
|
||||
"""
|
||||
global _unpinned_env
|
||||
launch_env = _swap_env(pins)
|
||||
if _unpinned_env is not None:
|
||||
return
|
||||
_unpinned_env = launch_env if _nccl_has_no_cache() else {}
|
||||
if _unpinned_env:
|
||||
no_cache = os.environ.get("NCCL_NO_CACHE")
|
||||
os.environ["NCCL_NO_CACHE"] = ",".join(filter(None, [no_cache, *pins]))
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def unpinned_nccl_env() -> Iterator[None]:
|
||||
"""Create communicators shared with other processes without the pins."""
|
||||
if _unpinned_env == {}:
|
||||
logger.warning_once(
|
||||
"NCCL_NO_CACHE needs CUDA NCCL >= 2.29.7, so the NCCL settings vLLM "
|
||||
"pinned also apply here; a process sharing this communicator must "
|
||||
"set the same NCCL_* variables or it will hang."
|
||||
)
|
||||
pinned = _swap_env(_unpinned_env or {})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
_swap_env(pinned)
|
||||
|
||||
|
||||
def _nccl_has_no_cache() -> bool:
|
||||
# NCCL_NO_CACHE arrived in NCCL 2.29.7; ask the library vLLM loads.
|
||||
if torch.version.cuda is None:
|
||||
return False
|
||||
version = ctypes.c_int()
|
||||
try:
|
||||
ctypes.CDLL(find_nccl_library()).ncclGetVersion(ctypes.byref(version))
|
||||
except Exception:
|
||||
return False
|
||||
return version.value >= 22907
|
||||
|
||||
|
||||
def _swap_env(values: Mapping[str, str | None]) -> dict[str, str | None]:
|
||||
"""Set (None: unset) the given variables and return their previous values."""
|
||||
old = {name: os.environ.get(name) for name in values}
|
||||
for name, value in values.items():
|
||||
if value is None:
|
||||
os.environ.pop(name, None)
|
||||
else:
|
||||
os.environ[name] = value
|
||||
return old
|
||||
|
||||
Reference in New Issue
Block a user