[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)

Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-08-07 00:35:59 -07:00
committed by GitHub
co-authored by Claude Opus 5
parent 43d691ec6b
commit 21ea5b4fa1
14 changed files with 1422 additions and 783 deletions
+473 -20
View File
@@ -7,6 +7,8 @@ Integration tests for NCCL and IPC weight transfer between processes using Ray.
"""
import pickle
import threading
import time
from unittest.mock import MagicMock
import pybase64 as base64
@@ -20,9 +22,11 @@ from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer import (
HTTPVLLMWeightSyncClient,
ModuleSource,
ParamMeta,
RayVLLMWeightSyncClient,
TrainerWeightTransferEngine,
VLLMWeightSyncClient,
WeightSource,
WeightTransferEngineFactory,
WeightTransferTrainerFactory,
)
@@ -39,11 +43,19 @@ from vllm.distributed.weight_transfer.ipc_engine import (
IPCWeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerInitInfo,
NCCLTrainerWeightTransferEngine,
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.packed_tensor import (
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
DEFAULT_PACKED_NUM_BUFFERS,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLTrainerInitInfo,
SparseNCCLTrainerWeightTransferEngine,
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
SparseWeightPatch,
@@ -535,11 +547,15 @@ def inference_receive_tensor(
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (joins as rank 1)
# Trainer broadcasts a single tensor unpacked, so the worker must not
# expect the packed wire format (packed is a must-agree wire param shipped
# on the init info).
init_info = NCCLWeightTransferInitInfo(
master_address=master_address,
master_port=master_port,
rank_offset=1, # Trainer is rank 0, we become rank 1
world_size=world_size,
packed=False,
)
engine.init_transfer_engine(init_info)
@@ -616,39 +632,53 @@ def trainer_broadcast_sparse_tensor(
master_port: int,
world_size: int,
) -> bool:
"""Trainer task that broadcasts sparse patches via NCCL."""
"""Trainer task that broadcasts sparse patches via the trainer engine.
The worker task drives its own init/receive directly (it is not an RPC
endpoint), so the engine gets a no-op control-plane client; the NCCL
rendezvous and the patch broadcasts are the real thing.
"""
import torch
device = _set_ray_assigned_device()
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
)
from vllm.distributed.weight_transfer import WeightTransferTrainerFactory
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLTrainerInitInfo,
SparseWeightPatch,
)
pg = StatelessProcessGroup.create(
host=master_address,
port=master_port,
rank=0,
world_size=world_size,
)
comm = PyNcclCommunicator(pg, device=device.index)
class NoopClient:
def init_weight_transfer_engine(self, init_info):
pass
def start_weight_update(self):
pass
def update_weights(self, update_info):
pass
def finish_weight_update(self):
pass
patch = SparseWeightPatch(
name="test.weight",
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
full_shape=(10, 10),
)
SparseNCCLWeightTransferEngine.trainer_send_weights(
iter([patch]),
NCCLTrainerSendWeightsArgs(group=comm),
engine = WeightTransferTrainerFactory.trainer_init(
init_info=SparseNCCLTrainerInitInfo(
master_address=master_address,
master_port=master_port,
world_size=world_size,
rank=0,
),
client=NoopClient(),
)
engine.send_weights([patch])
torch.accelerator.synchronize()
engine.shutdown()
return True
@@ -1369,14 +1399,26 @@ class TestModuleSource:
source = ModuleSource(_module_with(("w", torch.zeros(2))))
assert [n for n, _ in source] == [n for n, _ in source] == ["w"]
def test_metadata_agrees_with_iteration(self):
"""The two channels must line up element-for-element: engines declare
the round from `metadata()` and then send what iteration yields."""
source = ModuleSource(
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
)
meta = source.metadata()
pairs = list(source)
assert [m.name for m in meta] == [name for name, _ in pairs]
assert [m.dtype for m in meta] == [t.dtype for _, t in pairs]
assert [m.shape for m in meta] == [tuple(t.shape) for _, t in pairs]
class TestTrainerFactory:
"""WeightTransferTrainerFactory registry mechanics."""
def test_registry_has_ipc(self):
# IPC is the first backend migrated to the trainer engine; NCCL /
# sparse NCCL register in later PRs.
def test_registry_has_all_backends(self):
assert "nccl" in WeightTransferTrainerFactory._registry
assert "ipc" in WeightTransferTrainerFactory._registry
assert "sparse_nccl" in WeightTransferTrainerFactory._registry
def test_register_and_dispatch(self):
saved = dict(WeightTransferTrainerFactory._registry)
@@ -1406,6 +1448,12 @@ class TestTrainerFactory:
def test_ipc_init_info_declares_backend(self):
assert IPCTrainerInitInfo.backend == "ipc"
def test_nccl_init_info_declares_backend(self):
assert NCCLTrainerInitInfo.backend == "nccl"
def test_sparse_nccl_init_info_declares_backend(self):
assert SparseNCCLTrainerInitInfo.backend == "sparse_nccl"
def test_trainer_init_info_subclass_must_set_backend(self):
with pytest.raises(TypeError, match="class-level `backend`"):
@@ -1475,3 +1523,408 @@ def test_ipc_trainer_init_ships_packed_to_worker():
assert engine.packed is True
assert client.order == ["init"]
assert client.last_init_info == {"packed": True}
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
"""The sender's trainer_init drives the inference-side init handshake with
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
and propagates the must-agree wire params to the worker."""
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
# Bypass the real NCCL rendezvous.
monkeypatch.setattr(
nccl_engine_mod, "open_trainer_endpoint", lambda info: MagicMock()
)
client = RecordingClient()
engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=29500,
world_size=3,
rank=0,
packed=True,
packed_buffer_size_bytes=1024,
packed_num_buffers=3,
),
client=client,
source=ModuleSource(_module_with(("w", torch.zeros(4)))),
)
assert isinstance(engine, NCCLTrainerWeightTransferEngine)
assert engine.is_sender is True
assert engine.packed is True
assert client.order == ["init"]
assert client.last_init_info == {
"master_address": "127.0.0.1",
"master_port": 29500,
"rank_offset": 1,
"world_size": 3,
"packed": True,
"packed_buffer_size_bytes": 1024,
"packed_num_buffers": 3,
}
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
"""The worker engine reads packed + buffer geometry from the
trainer-supplied init info at the handshake, not from the config or the
per-round update info."""
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
monkeypatch.setattr(
nccl_engine_mod, "worker_init_process_group", lambda info, pc: MagicMock()
)
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda:0"),
MagicMock(spec=torch.nn.Module),
)
assert engine.packed is False # pre-handshake default (legacy unpacked)
engine.init_transfer_engine(
NCCLWeightTransferInitInfo(
master_address="127.0.0.1",
master_port=29500,
rank_offset=1,
world_size=2,
packed=True,
packed_buffer_size_bytes=2048,
packed_num_buffers=4,
)
)
assert engine.packed is True
assert engine.packed_buffer_size_bytes == 2048
assert engine.packed_num_buffers == 4
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
"""Non-sender trainer ranks build an engine without opening an endpoint or
touching the client; they only join the collectives in send_weights."""
client = RecordingClient()
engine = WeightTransferTrainerFactory.trainer_init(
init_info=NCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=29500,
world_size=3,
rank=1,
),
client=client,
source=ModuleSource(_module_with(("w", torch.zeros(4)))),
)
assert engine.is_sender is False
assert engine.model_update_group is None
assert client.order == []
# send_weights on a non-sender only iterates the source (packed mode needs
# no CUDA stream on non-senders), never the client.
engine.send_weights()
assert client.order == []
@pytest.mark.skipif(
torch.accelerator.device_count() < 1,
reason="Need at least 1 GPU (NCCL broadcast / CUDA stream).",
)
def test_nccl_trainer_send_weights_drives_client_in_order():
"""send_weights issues start -> update -> finish and ships per-round
metadata; the packed wire params ride the init handshake, not the
per-round update_info."""
client = RecordingClient()
engine = NCCLTrainerWeightTransferEngine(
client=client,
source=ModuleSource(_module_with(("w", torch.zeros(4, device="cuda")))),
packed=False,
)
# Bypass the real NCCL rendezvous; broadcast is a no-op.
engine.model_update_group = MagicMock()
engine.send_weights()
assert client.order == ["start", "update", "finish"]
assert client.last_update_info is not None
assert client.last_update_info["names"] == ["w"]
assert client.last_update_info["shapes"] == [[4]]
assert "packed" not in client.last_update_info
class _ScriptedSource(WeightSource):
"""Declares `meta` but yields whatever `pairs` says — used to drive the
metadata/iteration agreement checks."""
def __init__(self, meta, pairs):
self._meta = meta
self._pairs = pairs
def metadata(self):
return list(self._meta)
def __iter__(self):
yield from self._pairs
def _mock_group_engine(source, monkeypatch, **kwargs):
"""Unpacked trainer engine with a mocked group and stream (no GPU needed)."""
engine = NCCLTrainerWeightTransferEngine(
client=RecordingClient(), source=source, packed=False, **kwargs
)
engine.model_update_group = MagicMock()
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
return engine
def test_nccl_trainer_init_requires_source():
"""NCCL is a full-resync backend: it cannot run without a WeightSource."""
with pytest.raises(ValueError, match="requires a WeightSource"):
NCCLTrainerWeightTransferEngine.trainer_init(
NCCLTrainerInitInfo(
master_address="127.0.0.1", master_port=29500, world_size=2, rank=0
),
client=RecordingClient(),
)
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
iteration disagreeing with it must raise rather than corrupt the stream."""
meta = [
ParamMeta("w", torch.float32, (4,)),
ParamMeta("b", torch.float32, (2,)),
]
reordered = [("b", torch.zeros(2)), ("w", torch.zeros(4))]
engine = _mock_group_engine(_ScriptedSource(meta, reordered), monkeypatch)
with pytest.raises(ValueError, match="disagrees with iteration at index 0"):
engine.send_weights()
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
"""A source that declares one wire dtype and materializes another would make
the two sides disagree on every byte offset."""
meta = [ParamMeta("w", torch.float32, (4,))]
engine = _mock_group_engine(
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
monkeypatch,
)
with pytest.raises(ValueError, match="disagrees with iteration"):
engine.send_weights()
def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch):
"""Yielding fewer parameters than declared leaves the worker waiting."""
meta = [
ParamMeta("w", torch.float32, (4,)),
ParamMeta("b", torch.float32, (2,)),
]
engine = _mock_group_engine(
_ScriptedSource(meta, [("w", torch.zeros(4))]), monkeypatch
)
with pytest.raises(ValueError, match="yielded 1 parameters"):
engine.send_weights()
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
be linearized first or the worker receives unrelated memory."""
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
view = base.t() # non-contiguous
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
engine = _mock_group_engine(_ScriptedSource(meta, [("w", view)]), monkeypatch)
engine.send_weights()
sent = engine.model_update_group.broadcast.call_args.args[0]
assert sent.is_contiguous()
assert torch.equal(sent, view)
def test_nccl_trainer_send_weights_raises_instead_of_hanging(monkeypatch):
"""A failed broadcast must surface even while the inference-side
update_weights is still blocked in its matching NCCL call.
Joining the RPC thread there would deadlock: the worker only returns once
the broadcast it is waiting for arrives, which never happens.
"""
rpc_entered = threading.Event()
release_rpc = threading.Event()
class _BlockingClient(RecordingClient):
def update_weights(self, update_info):
rpc_entered.set()
release_rpc.wait(timeout=30) # stands in for a wedged NCCL recv
super().update_weights(update_info)
class _FailingSource(WeightSource):
def metadata(self):
return [ParamMeta("w", torch.float32, (4,))]
def __iter__(self):
# Fail only once the RPC is provably in flight.
rpc_entered.wait(timeout=60)
raise RuntimeError("broadcast blew up")
engine = NCCLTrainerWeightTransferEngine(
client=_BlockingClient(), source=_FailingSource(), packed=False
)
engine.model_update_group = MagicMock()
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
started = time.perf_counter()
try:
with pytest.raises(RuntimeError, match="broadcast blew up"):
engine.send_weights()
elapsed = time.perf_counter() - started
assert rpc_entered.is_set(), "the RPC was never in flight"
assert not release_rpc.is_set(), "send_weights waited for the wedged RPC"
# Regression guard: joining the RPC thread would park here until the
# client's own timeout expires instead of raising immediately.
assert elapsed < 5.0, f"send_weights blocked {elapsed:.1f}s on the RPC"
finally:
release_rpc.set()
def _sparse_patch(device: str = "cpu") -> SparseWeightPatch:
return SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32, device=device),
values=torch.tensor([1.0, 2.0], dtype=torch.float32, device=device),
full_shape=(4, 4),
)
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
"""The sender's trainer_init drives the init handshake with the
worker-shaped init info; sparse ships no packed wire params, so the worker
keeps its unpacked defaults. Sparse takes no `source`."""
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
client = RecordingClient()
engine = WeightTransferTrainerFactory.trainer_init(
init_info=SparseNCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=29500,
world_size=2,
rank=0,
),
client=client,
)
assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine)
assert client.order == ["init"]
assert client.last_init_info == {
"master_address": "127.0.0.1",
"master_port": 29500,
"rank_offset": 1,
"world_size": 2,
"packed": False,
"packed_buffer_size_bytes": DEFAULT_PACKED_BUFFER_SIZE_BYTES,
"packed_num_buffers": DEFAULT_PACKED_NUM_BUFFERS,
}
def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
"""send_weights takes the round's patches and ships per-patch metadata
(names / shapes / num_updates_list) + broadcasts indices + values each."""
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
# The group is a mock, so the stream is just a handle it is handed (and a
# handle _post_send_sync can synchronize).
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
engine.send_weights([_sparse_patch()])
assert client.order == ["start", "update", "finish"]
assert client.last_update_info is not None
assert client.last_update_info["names"] == ["w"]
assert client.last_update_info["shapes"] == [[4, 4]]
assert client.last_update_info["num_updates_list"] == [2]
# One broadcast for indices + one for values per patch.
assert engine.model_update_group.broadcast.call_count == 2
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
"""A round with no patches must not touch the client (an empty sparse
update info is invalid by construction)."""
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
engine.send_weights([])
engine.send_weights() # no argument is also a no-op round
assert client.order == []
def test_sparse_nccl_trainer_send_weights_requires_full_shape():
patch = _sparse_patch()
patch.full_shape = None
engine = SparseNCCLTrainerWeightTransferEngine(client=RecordingClient())
engine.model_update_group = MagicMock()
with pytest.raises(ValueError, match="full_shape"):
engine.send_weights([patch])
def test_sparse_nccl_trainer_rejects_source():
"""Sparse is a delta backend; a WeightSource would silently never be sent."""
with pytest.raises(ValueError, match="takes no WeightSource"):
SparseNCCLTrainerWeightTransferEngine(
client=RecordingClient(),
source=ModuleSource(_module_with(("w", torch.zeros(2)))),
)
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
"""Malformed patches must fail on the trainer, before start_weight_update:
the worker's own checks only run once the broadcasts are already in flight,
where a size mismatch wedges both sides instead of raising."""
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
mismatched = SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
full_shape=(4, 4),
)
with pytest.raises(ValueError, match="matching lengths"):
engine.send_weights([mismatched])
wrong_index_dtype = SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int64),
values=torch.tensor([1.0, 2.0], dtype=torch.float32),
full_shape=(4, 4),
)
with pytest.raises(ValueError, match="int32 indices"):
engine.send_weights([wrong_index_dtype])
assert client.order == []
def test_sparse_nccl_trainer_non_sender_skips_client():
client = RecordingClient()
engine = WeightTransferTrainerFactory.trainer_init(
init_info=SparseNCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=29500,
world_size=2,
rank=1,
),
client=client,
)
assert engine.is_sender is False
assert engine.model_update_group is None
assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine)
engine.send_weights([_sparse_patch()])
assert client.order == []