[RL] Support checkpoint-coordinate sparse NCCL weight updates (#53751)

Signed-off-by: aoshen02 <aoshen@inferact.ai>
Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com>
Co-authored-by: aoshen02 <aoshen@inferact.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Shuolei Wang
2026-08-27 18:19:51 +08:00
committed by GitHub
co-authored by aoshen02 OpenAI Codex
parent 50708f95ae
commit 6c4aa8bec3
5 changed files with 435 additions and 797 deletions
+120 -237
View File
@@ -9,6 +9,7 @@ Integration tests for NCCL and IPC weight transfer between processes using Ray.
import pickle
import threading
import time
from contextlib import nullcontext
from unittest.mock import MagicMock
import pybase64 as base64
@@ -313,103 +314,6 @@ class TestEngineRegistry:
)
# --- Unit Tests: Sparse patch application (CPU) ---
class TestSparseNCCLPatchApplication:
"""Test SparseNCCLWeightTransferEngine._apply_patch on a real param."""
def _make_engine(self, model):
config = WeightTransferConfig(backend="sparse_nccl")
return SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), torch.device("cpu"), model
)
def _make_model(self, numel: int = 8):
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False)
)
def get_parameter(name):
assert name == "w"
return model.w
model.get_parameter = get_parameter
return model
def test_apply_patch_updates_only_selected_entries(self):
model = self._make_model(8)
engine = self._make_engine(model)
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0, 7.0], dtype=torch.float32),
)
)
expected = torch.zeros(8)
expected[1] = 5.0
expected[3] = 7.0
assert torch.equal(model.w.data, expected)
def test_apply_patch_rejects_mismatched_lengths(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="matching lengths"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_non_int32_indices(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="int32 indices"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int64),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_dtype_mismatch(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="does not match"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.bfloat16),
)
)
def test_apply_patch_rejects_non_contiguous_param(self):
model = torch.nn.Module()
model.register_parameter(
"w",
torch.nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t(),
requires_grad=False,
),
)
model.get_parameter = lambda name: model.w
engine = self._make_engine(model)
with pytest.raises(NotImplementedError, match="contiguous params"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
)
# --- Test receive_weights without init raises ---
@@ -628,162 +532,135 @@ def test_nccl_weight_transfer_between_processes():
)
@ray.remote(num_gpus=1)
def trainer_broadcast_sparse_tensor(
master_address: str,
master_port: int,
world_size: int,
) -> bool:
"""Trainer task that broadcasts sparse patches via the trainer engine.
def test_sparse_nccl_checkpoint_chunks_to_ep_local_experts_cpu(monkeypatch):
"""Replay global expert patches through two EP-local loaders on CPU."""
The worker task drives its own init/receive directly (it is not an RPC
endpoint), so the engine gets a no-op control-plane client; the NCCL
rendezvous and the patch broadcasts are the real thing.
"""
import torch
device = _set_ray_assigned_device()
from vllm.distributed.weight_transfer import WeightTransferTrainerFactory
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLTrainerInitInfo,
SparseWeightPatch,
)
class NoopClient:
def init_weight_transfer_engine(self, init_info):
pass
class CaptureClient:
def __init__(self):
self.order: list[str] = []
self.update_infos: list[dict] = []
def start_weight_update(self):
pass
self.order.append("start")
def update_weights(self, update_info):
pass
self.order.append("update")
self.update_infos.append(update_info)
def finish_weight_update(self):
pass
self.order.append("finish")
patch = SparseWeightPatch(
name="test.weight",
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
full_shape=(10, 10),
)
engine = WeightTransferTrainerFactory.trainer_init(
init_info=SparseNCCLTrainerInitInfo(
master_address=master_address,
master_port=master_port,
world_size=world_size,
rank=0,
),
client=NoopClient(),
)
engine.send_weights([patch])
torch.accelerator.synchronize()
engine.shutdown()
return True
monkeypatch.setattr(torch.cuda, "current_stream", lambda *_, **__: None)
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
monkeypatch.setattr(torch.accelerator, "device_index", lambda *_: nullcontext())
@ray.remote(num_gpus=1)
def inference_receive_sparse_tensor(
master_address: str,
master_port: int,
world_size: int,
) -> dict:
"""Inference task that receives sparse patches via the sparse engine."""
from unittest.mock import MagicMock
import torch
device = _set_ray_assigned_device()
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
client = CaptureClient()
sender = SparseNCCLTrainerWeightTransferEngine(client=client)
sender.model_update_group = MagicMock()
sender.model_update_group.device = torch.device("cpu")
wire_payloads = []
sender.model_update_group.broadcast.side_effect = lambda tensor, **_: (
wire_payloads.append(tensor.clone())
)
config = WeightTransferConfig(backend="sparse_nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
# Real module holding the target parameter the patch will modify.
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False)
)
model.get_parameter = lambda name: model.w
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
)
engine = SparseNCCLWeightTransferEngine(
config, vllm_config, torch.device("cuda"), model
)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
)
engine.init_transfer_engine(
NCCLWeightTransferInitInfo(
master_address=master_address,
master_port=master_port,
rank_offset=1,
world_size=world_size,
expert_names = [
"model.layers.0.mlp.experts.0.gate_proj.weight",
"model.layers.0.mlp.experts.1.gate_proj.weight",
]
client.start_weight_update()
for name, index, value in zip(
expert_names,
(0, 3),
(5.0, 7.0),
strict=True,
):
sender.send_weight_chunk(
[
SparseWeightPatch(
name=name,
indices=torch.tensor([index], dtype=torch.int32),
values=torch.tensor([value]),
full_shape=(2, 2),
)
]
)
client.finish_weight_update()
assert client.order == ["start", "update", "update", "finish"]
assert [info["names"] for info in client.update_infos] == [
[expert_names[0]],
[expert_names[1]],
]
assert [info["shapes"] for info in client.update_infos] == [
[[2, 2]],
[[2, 2]],
]
assert [info["num_updates_list"] for info in client.update_infos] == [
[1],
[1],
]
expected_gates = (
[[5.0, -1.0], [-1.0, -1.0]],
[[-1.0, -1.0], [-1.0, 7.0]],
)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
for ep_rank, expected_gate in enumerate(expected_gates):
model = torch.nn.Module()
model.register_parameter(
"w13_weight",
torch.nn.Parameter(torch.full((1, 4, 2), -1.0), requires_grad=False),
)
local_name = expert_names[ep_rank]
seen_names: list[str] = []
loaded_names: set[str] = set()
copy_count = [0]
expected = torch.zeros(30, dtype=torch.float32, device=device)
expected[[1, 7, 25]] = torch.tensor(
[10.0, 20.0, 30.0], dtype=torch.float32, device=device
)
success = torch.equal(model.w.data, expected)
engine.shutdown()
return {
"success": success,
"selected_values": model.w.data[[1, 7, 25]].cpu().tolist(),
}
def load_weights(
weights,
local_name=local_name,
target=model,
seen=seen_names,
loaded=loaded_names,
count=copy_count,
):
for name, checkpoint_weight in weights:
seen.append(name)
if name != local_name:
continue
target.w13_weight.data[0, :2].copy_(checkpoint_weight)
loaded.add(name)
count[0] += 1
return loaded
model.load_weights = load_weights
receiver = SparseNCCLWeightTransferEngine(
WeightTransferConfig(backend="sparse_nccl"),
create_mock_vllm_config(rank=ep_rank, world_size=2),
torch.device("cpu"),
model,
)
receiver.model_update_group = MagicMock()
receiver.model_update_group.device = torch.device("cpu")
payloads = iter(wire_payloads)
receiver.model_update_group.broadcast.side_effect = (
lambda tensor, payloads=payloads, **_: tensor.copy_(next(payloads))
)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL sparse weight transfer test.",
)
def test_nccl_sparse_weight_transfer_between_processes():
"""Test NCCL sparse weight transfer from trainer to inference process."""
_init_ray_for_weight_transfer()
receiver.start_weight_update()
for update_info in client.update_infos:
receiver.receive_weights(SparseNCCLWeightTransferUpdateInfo(**update_info))
receiver.finish_weight_update()
master_address = "127.0.0.1"
master_port = get_open_port()
world_size = 2
assert seen_names == expert_names
assert loaded_names == {local_name}
assert copy_count == [1]
assert model.w13_weight[0, :2].tolist() == expected_gate
assert model.w13_weight[0, 2:].eq(-1).all()
foreign_value = 7.0 if ep_rank == 0 else 5.0
assert not model.w13_weight.eq(foreign_value).any()
receiver.shutdown()
inference_future = inference_receive_sparse_tensor.remote(
master_address, master_port, world_size
)
trainer_future = trainer_broadcast_sparse_tensor.remote(
master_address, master_port, world_size
)
trainer_result, result = ray.get([trainer_future, inference_future])
assert trainer_result, "Trainer should complete successfully"
assert result["success"], (
"Sparse weight transfer failed. "
f"Received selected values: {result['selected_values']}"
)
sender.shutdown()
# --- Unit Tests: IPCWeightTransferUpdateInfo Validation ---
@@ -1995,11 +1872,11 @@ def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
"""send_weights takes the round's patches and ships per-patch metadata
(names / shapes / num_updates_list) + broadcasts indices + values each."""
"""One-shot send drives start, update, and finish in order."""
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
engine.model_update_group.device = torch.device("cpu")
# The group is a mock, so the stream is just a handle it is handed (and a
# handle _post_send_sync can synchronize).
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
@@ -2013,6 +1890,7 @@ def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
assert client.last_update_info["num_updates_list"] == [2]
# One broadcast for indices + one for values per patch.
assert engine.model_update_group.broadcast.call_count == 2
engine.shutdown()
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
@@ -2024,6 +1902,8 @@ def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
engine.send_weights([])
engine.send_weights() # no argument is also a no-op round
engine.send_weight_chunk([])
engine.send_weight_chunk()
assert client.order == []
@@ -2033,6 +1913,7 @@ def test_sparse_nccl_trainer_send_weights_requires_full_shape():
patch.full_shape = None
engine = SparseNCCLTrainerWeightTransferEngine(client=RecordingClient())
engine.model_update_group = MagicMock()
engine.model_update_group.device = torch.device("cpu")
with pytest.raises(ValueError, match="full_shape"):
engine.send_weights([patch])
@@ -2054,6 +1935,7 @@ def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
client = RecordingClient()
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
engine.model_update_group = MagicMock()
engine.model_update_group.device = torch.device("cpu")
mismatched = SparseWeightPatch(
name="w",
@@ -2092,4 +1974,5 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
assert engine.model_update_group is None
assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine)
engine.send_weights([_sparse_patch()])
engine.send_weight_chunk([_sparse_patch()])
assert client.order == []