[rl] Stateful Trainer Send: NCCL + Sparse NCCL [3/N] (#50902)
Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
43d691ec6b
commit
21ea5b4fa1
@@ -7,6 +7,8 @@ Integration tests for NCCL and IPC weight transfer between processes using Ray.
|
||||
"""
|
||||
|
||||
import pickle
|
||||
import threading
|
||||
import time
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
@@ -20,9 +22,11 @@ from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import (
|
||||
HTTPVLLMWeightSyncClient,
|
||||
ModuleSource,
|
||||
ParamMeta,
|
||||
RayVLLMWeightSyncClient,
|
||||
TrainerWeightTransferEngine,
|
||||
VLLMWeightSyncClient,
|
||||
WeightSource,
|
||||
WeightTransferEngineFactory,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
@@ -39,11 +43,19 @@ from vllm.distributed.weight_transfer.ipc_engine import (
|
||||
IPCWeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerInitInfo,
|
||||
NCCLTrainerWeightTransferEngine,
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
DEFAULT_PACKED_NUM_BUFFERS,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLTrainerInitInfo,
|
||||
SparseNCCLTrainerWeightTransferEngine,
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLWeightTransferUpdateInfo,
|
||||
SparseWeightPatch,
|
||||
@@ -535,11 +547,15 @@ def inference_receive_tensor(
|
||||
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
|
||||
|
||||
# Initialize the engine (joins as rank 1)
|
||||
# Trainer broadcasts a single tensor unpacked, so the worker must not
|
||||
# expect the packed wire format (packed is a must-agree wire param shipped
|
||||
# on the init info).
|
||||
init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1, # Trainer is rank 0, we become rank 1
|
||||
world_size=world_size,
|
||||
packed=False,
|
||||
)
|
||||
engine.init_transfer_engine(init_info)
|
||||
|
||||
@@ -616,39 +632,53 @@ def trainer_broadcast_sparse_tensor(
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
) -> bool:
|
||||
"""Trainer task that broadcasts sparse patches via NCCL."""
|
||||
"""Trainer task that broadcasts sparse patches via the trainer engine.
|
||||
|
||||
The worker task drives its own init/receive directly (it is not an RPC
|
||||
endpoint), so the engine gets a no-op control-plane client; the NCCL
|
||||
rendezvous and the patch broadcasts are the real thing.
|
||||
"""
|
||||
import torch
|
||||
|
||||
device = _set_ray_assigned_device()
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
)
|
||||
from vllm.distributed.weight_transfer import WeightTransferTrainerFactory
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLTrainerInitInfo,
|
||||
SparseWeightPatch,
|
||||
)
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address,
|
||||
port=master_port,
|
||||
rank=0,
|
||||
world_size=world_size,
|
||||
)
|
||||
comm = PyNcclCommunicator(pg, device=device.index)
|
||||
class NoopClient:
|
||||
def init_weight_transfer_engine(self, init_info):
|
||||
pass
|
||||
|
||||
def start_weight_update(self):
|
||||
pass
|
||||
|
||||
def update_weights(self, update_info):
|
||||
pass
|
||||
|
||||
def finish_weight_update(self):
|
||||
pass
|
||||
|
||||
patch = SparseWeightPatch(
|
||||
name="test.weight",
|
||||
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
|
||||
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
|
||||
full_shape=(10, 10),
|
||||
)
|
||||
SparseNCCLWeightTransferEngine.trainer_send_weights(
|
||||
iter([patch]),
|
||||
NCCLTrainerSendWeightsArgs(group=comm),
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=SparseNCCLTrainerInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
world_size=world_size,
|
||||
rank=0,
|
||||
),
|
||||
client=NoopClient(),
|
||||
)
|
||||
engine.send_weights([patch])
|
||||
torch.accelerator.synchronize()
|
||||
engine.shutdown()
|
||||
return True
|
||||
|
||||
|
||||
@@ -1369,14 +1399,26 @@ class TestModuleSource:
|
||||
source = ModuleSource(_module_with(("w", torch.zeros(2))))
|
||||
assert [n for n, _ in source] == [n for n, _ in source] == ["w"]
|
||||
|
||||
def test_metadata_agrees_with_iteration(self):
|
||||
"""The two channels must line up element-for-element: engines declare
|
||||
the round from `metadata()` and then send what iteration yields."""
|
||||
source = ModuleSource(
|
||||
_module_with(("w", torch.zeros(2, 3)), ("b", torch.zeros(3)))
|
||||
)
|
||||
meta = source.metadata()
|
||||
pairs = list(source)
|
||||
assert [m.name for m in meta] == [name for name, _ in pairs]
|
||||
assert [m.dtype for m in meta] == [t.dtype for _, t in pairs]
|
||||
assert [m.shape for m in meta] == [tuple(t.shape) for _, t in pairs]
|
||||
|
||||
|
||||
class TestTrainerFactory:
|
||||
"""WeightTransferTrainerFactory registry mechanics."""
|
||||
|
||||
def test_registry_has_ipc(self):
|
||||
# IPC is the first backend migrated to the trainer engine; NCCL /
|
||||
# sparse NCCL register in later PRs.
|
||||
def test_registry_has_all_backends(self):
|
||||
assert "nccl" in WeightTransferTrainerFactory._registry
|
||||
assert "ipc" in WeightTransferTrainerFactory._registry
|
||||
assert "sparse_nccl" in WeightTransferTrainerFactory._registry
|
||||
|
||||
def test_register_and_dispatch(self):
|
||||
saved = dict(WeightTransferTrainerFactory._registry)
|
||||
@@ -1406,6 +1448,12 @@ class TestTrainerFactory:
|
||||
def test_ipc_init_info_declares_backend(self):
|
||||
assert IPCTrainerInitInfo.backend == "ipc"
|
||||
|
||||
def test_nccl_init_info_declares_backend(self):
|
||||
assert NCCLTrainerInitInfo.backend == "nccl"
|
||||
|
||||
def test_sparse_nccl_init_info_declares_backend(self):
|
||||
assert SparseNCCLTrainerInitInfo.backend == "sparse_nccl"
|
||||
|
||||
def test_trainer_init_info_subclass_must_set_backend(self):
|
||||
with pytest.raises(TypeError, match="class-level `backend`"):
|
||||
|
||||
@@ -1475,3 +1523,408 @@ def test_ipc_trainer_init_ships_packed_to_worker():
|
||||
assert engine.packed is True
|
||||
assert client.order == ["init"]
|
||||
assert client.last_init_info == {"packed": True}
|
||||
|
||||
|
||||
def test_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
"""The sender's trainer_init drives the inference-side init handshake with
|
||||
the worker-shaped init info (rank_offset=1) while opening its own endpoint,
|
||||
and propagates the must-agree wire params to the worker."""
|
||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||
|
||||
# Bypass the real NCCL rendezvous.
|
||||
monkeypatch.setattr(
|
||||
nccl_engine_mod, "open_trainer_endpoint", lambda info: MagicMock()
|
||||
)
|
||||
|
||||
client = RecordingClient()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=29500,
|
||||
world_size=3,
|
||||
rank=0,
|
||||
packed=True,
|
||||
packed_buffer_size_bytes=1024,
|
||||
packed_num_buffers=3,
|
||||
),
|
||||
client=client,
|
||||
source=ModuleSource(_module_with(("w", torch.zeros(4)))),
|
||||
)
|
||||
|
||||
assert isinstance(engine, NCCLTrainerWeightTransferEngine)
|
||||
assert engine.is_sender is True
|
||||
assert engine.packed is True
|
||||
assert client.order == ["init"]
|
||||
assert client.last_init_info == {
|
||||
"master_address": "127.0.0.1",
|
||||
"master_port": 29500,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
"packed": True,
|
||||
"packed_buffer_size_bytes": 1024,
|
||||
"packed_num_buffers": 3,
|
||||
}
|
||||
|
||||
|
||||
def test_nccl_worker_learns_wire_params_from_init_handshake(monkeypatch):
|
||||
"""The worker engine reads packed + buffer geometry from the
|
||||
trainer-supplied init info at the handshake, not from the config or the
|
||||
per-round update info."""
|
||||
import vllm.distributed.weight_transfer.nccl_engine as nccl_engine_mod
|
||||
|
||||
monkeypatch.setattr(
|
||||
nccl_engine_mod, "worker_init_process_group", lambda info, pc: MagicMock()
|
||||
)
|
||||
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda:0"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
assert engine.packed is False # pre-handshake default (legacy unpacked)
|
||||
engine.init_transfer_engine(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=29500,
|
||||
rank_offset=1,
|
||||
world_size=2,
|
||||
packed=True,
|
||||
packed_buffer_size_bytes=2048,
|
||||
packed_num_buffers=4,
|
||||
)
|
||||
)
|
||||
|
||||
assert engine.packed is True
|
||||
assert engine.packed_buffer_size_bytes == 2048
|
||||
assert engine.packed_num_buffers == 4
|
||||
|
||||
|
||||
def test_nccl_trainer_init_non_sender_skips_rendezvous_and_client():
|
||||
"""Non-sender trainer ranks build an engine without opening an endpoint or
|
||||
touching the client; they only join the collectives in send_weights."""
|
||||
client = RecordingClient()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=NCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=29500,
|
||||
world_size=3,
|
||||
rank=1,
|
||||
),
|
||||
client=client,
|
||||
source=ModuleSource(_module_with(("w", torch.zeros(4)))),
|
||||
)
|
||||
|
||||
assert engine.is_sender is False
|
||||
assert engine.model_update_group is None
|
||||
assert client.order == []
|
||||
|
||||
# send_weights on a non-sender only iterates the source (packed mode needs
|
||||
# no CUDA stream on non-senders), never the client.
|
||||
engine.send_weights()
|
||||
assert client.order == []
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 1,
|
||||
reason="Need at least 1 GPU (NCCL broadcast / CUDA stream).",
|
||||
)
|
||||
def test_nccl_trainer_send_weights_drives_client_in_order():
|
||||
"""send_weights issues start -> update -> finish and ships per-round
|
||||
metadata; the packed wire params ride the init handshake, not the
|
||||
per-round update_info."""
|
||||
client = RecordingClient()
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=client,
|
||||
source=ModuleSource(_module_with(("w", torch.zeros(4, device="cuda")))),
|
||||
packed=False,
|
||||
)
|
||||
# Bypass the real NCCL rendezvous; broadcast is a no-op.
|
||||
engine.model_update_group = MagicMock()
|
||||
|
||||
engine.send_weights()
|
||||
|
||||
assert client.order == ["start", "update", "finish"]
|
||||
assert client.last_update_info is not None
|
||||
assert client.last_update_info["names"] == ["w"]
|
||||
assert client.last_update_info["shapes"] == [[4]]
|
||||
assert "packed" not in client.last_update_info
|
||||
|
||||
|
||||
class _ScriptedSource(WeightSource):
|
||||
"""Declares `meta` but yields whatever `pairs` says — used to drive the
|
||||
metadata/iteration agreement checks."""
|
||||
|
||||
def __init__(self, meta, pairs):
|
||||
self._meta = meta
|
||||
self._pairs = pairs
|
||||
|
||||
def metadata(self):
|
||||
return list(self._meta)
|
||||
|
||||
def __iter__(self):
|
||||
yield from self._pairs
|
||||
|
||||
|
||||
def _mock_group_engine(source, monkeypatch, **kwargs):
|
||||
"""Unpacked trainer engine with a mocked group and stream (no GPU needed)."""
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=RecordingClient(), source=source, packed=False, **kwargs
|
||||
)
|
||||
engine.model_update_group = MagicMock()
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
|
||||
return engine
|
||||
|
||||
|
||||
def test_nccl_trainer_init_requires_source():
|
||||
"""NCCL is a full-resync backend: it cannot run without a WeightSource."""
|
||||
with pytest.raises(ValueError, match="requires a WeightSource"):
|
||||
NCCLTrainerWeightTransferEngine.trainer_init(
|
||||
NCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1", master_port=29500, world_size=2, rank=0
|
||||
),
|
||||
client=RecordingClient(),
|
||||
)
|
||||
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_reordered_source(monkeypatch):
|
||||
"""The worker sizes its buffers (and cuts packed chunks) from metadata(), so
|
||||
iteration disagreeing with it must raise rather than corrupt the stream."""
|
||||
meta = [
|
||||
ParamMeta("w", torch.float32, (4,)),
|
||||
ParamMeta("b", torch.float32, (2,)),
|
||||
]
|
||||
reordered = [("b", torch.zeros(2)), ("w", torch.zeros(4))]
|
||||
engine = _mock_group_engine(_ScriptedSource(meta, reordered), monkeypatch)
|
||||
|
||||
with pytest.raises(ValueError, match="disagrees with iteration at index 0"):
|
||||
engine.send_weights()
|
||||
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_dtype_disagreement(monkeypatch):
|
||||
"""A source that declares one wire dtype and materializes another would make
|
||||
the two sides disagree on every byte offset."""
|
||||
meta = [ParamMeta("w", torch.float32, (4,))]
|
||||
engine = _mock_group_engine(
|
||||
_ScriptedSource(meta, [("w", torch.zeros(4, dtype=torch.bfloat16))]),
|
||||
monkeypatch,
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="disagrees with iteration"):
|
||||
engine.send_weights()
|
||||
|
||||
|
||||
def test_nccl_trainer_send_weights_rejects_truncated_source(monkeypatch):
|
||||
"""Yielding fewer parameters than declared leaves the worker waiting."""
|
||||
meta = [
|
||||
ParamMeta("w", torch.float32, (4,)),
|
||||
ParamMeta("b", torch.float32, (2,)),
|
||||
]
|
||||
engine = _mock_group_engine(
|
||||
_ScriptedSource(meta, [("w", torch.zeros(4))]), monkeypatch
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="yielded 1 parameters"):
|
||||
engine.send_weights()
|
||||
|
||||
|
||||
def test_nccl_trainer_send_weights_broadcasts_contiguous(monkeypatch):
|
||||
"""NCCL sends numel elements from data_ptr(), so a non-contiguous view must
|
||||
be linearized first or the worker receives unrelated memory."""
|
||||
base = torch.arange(6, dtype=torch.float32).reshape(2, 3)
|
||||
view = base.t() # non-contiguous
|
||||
meta = [ParamMeta("w", torch.float32, tuple(view.shape))]
|
||||
engine = _mock_group_engine(_ScriptedSource(meta, [("w", view)]), monkeypatch)
|
||||
|
||||
engine.send_weights()
|
||||
|
||||
sent = engine.model_update_group.broadcast.call_args.args[0]
|
||||
assert sent.is_contiguous()
|
||||
assert torch.equal(sent, view)
|
||||
|
||||
|
||||
def test_nccl_trainer_send_weights_raises_instead_of_hanging(monkeypatch):
|
||||
"""A failed broadcast must surface even while the inference-side
|
||||
update_weights is still blocked in its matching NCCL call.
|
||||
|
||||
Joining the RPC thread there would deadlock: the worker only returns once
|
||||
the broadcast it is waiting for arrives, which never happens.
|
||||
"""
|
||||
rpc_entered = threading.Event()
|
||||
release_rpc = threading.Event()
|
||||
|
||||
class _BlockingClient(RecordingClient):
|
||||
def update_weights(self, update_info):
|
||||
rpc_entered.set()
|
||||
release_rpc.wait(timeout=30) # stands in for a wedged NCCL recv
|
||||
super().update_weights(update_info)
|
||||
|
||||
class _FailingSource(WeightSource):
|
||||
def metadata(self):
|
||||
return [ParamMeta("w", torch.float32, (4,))]
|
||||
|
||||
def __iter__(self):
|
||||
# Fail only once the RPC is provably in flight.
|
||||
rpc_entered.wait(timeout=60)
|
||||
raise RuntimeError("broadcast blew up")
|
||||
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=_BlockingClient(), source=_FailingSource(), packed=False
|
||||
)
|
||||
engine.model_update_group = MagicMock()
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
|
||||
|
||||
started = time.perf_counter()
|
||||
try:
|
||||
with pytest.raises(RuntimeError, match="broadcast blew up"):
|
||||
engine.send_weights()
|
||||
elapsed = time.perf_counter() - started
|
||||
assert rpc_entered.is_set(), "the RPC was never in flight"
|
||||
assert not release_rpc.is_set(), "send_weights waited for the wedged RPC"
|
||||
# Regression guard: joining the RPC thread would park here until the
|
||||
# client's own timeout expires instead of raising immediately.
|
||||
assert elapsed < 5.0, f"send_weights blocked {elapsed:.1f}s on the RPC"
|
||||
finally:
|
||||
release_rpc.set()
|
||||
|
||||
|
||||
def _sparse_patch(device: str = "cpu") -> SparseWeightPatch:
|
||||
return SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int32, device=device),
|
||||
values=torch.tensor([1.0, 2.0], dtype=torch.float32, device=device),
|
||||
full_shape=(4, 4),
|
||||
)
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
"""The sender's trainer_init drives the init handshake with the
|
||||
worker-shaped init info; sparse ships no packed wire params, so the worker
|
||||
keeps its unpacked defaults. Sparse takes no `source`."""
|
||||
import vllm.distributed.weight_transfer.sparse_nccl_engine as sparse_mod
|
||||
|
||||
monkeypatch.setattr(sparse_mod, "open_trainer_endpoint", lambda info: MagicMock())
|
||||
|
||||
client = RecordingClient()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=SparseNCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=29500,
|
||||
world_size=2,
|
||||
rank=0,
|
||||
),
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine)
|
||||
assert client.order == ["init"]
|
||||
assert client.last_init_info == {
|
||||
"master_address": "127.0.0.1",
|
||||
"master_port": 29500,
|
||||
"rank_offset": 1,
|
||||
"world_size": 2,
|
||||
"packed": False,
|
||||
"packed_buffer_size_bytes": DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
"packed_num_buffers": DEFAULT_PACKED_NUM_BUFFERS,
|
||||
}
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
|
||||
"""send_weights takes the round's patches and ships per-patch metadata
|
||||
(names / shapes / num_updates_list) + broadcasts indices + values each."""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
# The group is a mock, so the stream is just a handle it is handed (and a
|
||||
# handle _post_send_sync can synchronize).
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
|
||||
|
||||
engine.send_weights([_sparse_patch()])
|
||||
|
||||
assert client.order == ["start", "update", "finish"]
|
||||
assert client.last_update_info is not None
|
||||
assert client.last_update_info["names"] == ["w"]
|
||||
assert client.last_update_info["shapes"] == [[4, 4]]
|
||||
assert client.last_update_info["num_updates_list"] == [2]
|
||||
# One broadcast for indices + one for values per patch.
|
||||
assert engine.model_update_group.broadcast.call_count == 2
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
||||
"""A round with no patches must not touch the client (an empty sparse
|
||||
update info is invalid by construction)."""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
|
||||
engine.send_weights([])
|
||||
engine.send_weights() # no argument is also a no-op round
|
||||
|
||||
assert client.order == []
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_requires_full_shape():
|
||||
patch = _sparse_patch()
|
||||
patch.full_shape = None
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=RecordingClient())
|
||||
engine.model_update_group = MagicMock()
|
||||
|
||||
with pytest.raises(ValueError, match="full_shape"):
|
||||
engine.send_weights([patch])
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_rejects_source():
|
||||
"""Sparse is a delta backend; a WeightSource would silently never be sent."""
|
||||
with pytest.raises(ValueError, match="takes no WeightSource"):
|
||||
SparseNCCLTrainerWeightTransferEngine(
|
||||
client=RecordingClient(),
|
||||
source=ModuleSource(_module_with(("w", torch.zeros(2)))),
|
||||
)
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
|
||||
"""Malformed patches must fail on the trainer, before start_weight_update:
|
||||
the worker's own checks only run once the broadcasts are already in flight,
|
||||
where a size mismatch wedges both sides instead of raising."""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
|
||||
mismatched = SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
full_shape=(4, 4),
|
||||
)
|
||||
with pytest.raises(ValueError, match="matching lengths"):
|
||||
engine.send_weights([mismatched])
|
||||
|
||||
wrong_index_dtype = SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int64),
|
||||
values=torch.tensor([1.0, 2.0], dtype=torch.float32),
|
||||
full_shape=(4, 4),
|
||||
)
|
||||
with pytest.raises(ValueError, match="int32 indices"):
|
||||
engine.send_weights([wrong_index_dtype])
|
||||
|
||||
assert client.order == []
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_non_sender_skips_client():
|
||||
client = RecordingClient()
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=SparseNCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=29500,
|
||||
world_size=2,
|
||||
rank=1,
|
||||
),
|
||||
client=client,
|
||||
)
|
||||
|
||||
assert engine.is_sender is False
|
||||
assert engine.model_update_group is None
|
||||
assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine)
|
||||
engine.send_weights([_sparse_patch()])
|
||||
assert client.order == []
|
||||
|
||||
Reference in New Issue
Block a user