[Bugfix] Keep batch-invariance NCCL pins out of the weight-transfer group (#59500)

Signed-off-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: aoshen524 <aoshen524@gmail.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
This commit is contained in:
Guanxin Li
2026-10-02 11:19:28 +08:00
committed by GitHub
co-authored by aoshen524 Claude Opus 5.5 aoshen02
parent fcfacc3499
commit a810316c08
4 changed files with 140 additions and 27 deletions
+43 -5
View File
@@ -68,6 +68,7 @@ from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseWeightPatch,
)
from vllm.platforms import current_platform
from vllm.utils.nccl import _nccl_has_no_cache
from vllm.utils.network_utils import get_open_port
@@ -433,13 +434,15 @@ def trainer_broadcast_tensor(
return True
@ray.remote(num_gpus=1)
# max_calls=1: a batch-invariant run leaves NCCL pins in the worker process.
@ray.remote(num_gpus=1, max_calls=1)
def inference_receive_tensor(
master_address: str,
master_port: int,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
batch_invariant: bool = False,
) -> dict:
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
import contextlib
@@ -448,6 +451,21 @@ def inference_receive_tensor(
import torch
_set_ray_assigned_device()
if batch_invariant:
from vllm.model_executor.determinism.batch_invariant import (
override_envs_for_invariance,
)
override_envs_for_invariance()
# Like vLLM's own groups, a pinned communicator makes NCCL read the
# pins before the transfer group exists.
torch.distributed.init_process_group(
"nccl",
init_method=f"tcp://127.0.0.1:{get_open_port()}",
rank=0,
world_size=1,
)
torch.distributed.all_reduce(torch.ones(1, device="cuda"))
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
@@ -534,11 +552,25 @@ def inference_receive_tensor(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_between_processes():
@pytest.mark.parametrize(
"batch_invariant",
[
pytest.param(False, id="default"),
pytest.param(
True,
id="batch-invariant-worker",
marks=pytest.mark.skipif(
not _nccl_has_no_cache(), reason="Needs CUDA NCCL >= 2.29.7."
),
),
],
)
def test_nccl_weight_transfer_between_processes(batch_invariant):
"""Test NCCL weight transfer from trainer to inference process using Ray.
This test verifies that the NCCLWeightTransferEngine can receive
tensors broadcast by a trainer process via NCCL.
tensors broadcast by a trainer process via NCCL, including when the
worker runs with batch-invariance NCCL pins the trainer does not have.
"""
_init_ray_for_weight_transfer()
@@ -550,13 +582,19 @@ def test_nccl_weight_transfer_between_processes():
tensor_dtype = "float32"
inference_future = inference_receive_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
master_address,
master_port,
world_size,
tensor_shape,
tensor_dtype,
batch_invariant,
)
trainer_future = trainer_broadcast_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
trainer_result, result = ray.get([trainer_future, inference_future])
# A mismatched NCCL config deadlocks instead of failing.
trainer_result, result = ray.get([trainer_future, inference_future], timeout=300)
assert trainer_result, "Trainer should complete successfully"
assert result["success"], (