[RL] Support checkpoint-coordinate sparse NCCL weight updates (#53751)
Signed-off-by: aoshen02 <aoshen@inferact.ai> Signed-off-by: Shuolei Wang <shuoleiwang123@gmail.com> Co-authored-by: aoshen02 <aoshen@inferact.ai> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
aoshen02
OpenAI Codex
parent
50708f95ae
commit
6c4aa8bec3
@@ -9,6 +9,7 @@ Integration tests for NCCL and IPC weight transfer between processes using Ray.
|
||||
import pickle
|
||||
import threading
|
||||
import time
|
||||
from contextlib import nullcontext
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
@@ -313,103 +314,6 @@ class TestEngineRegistry:
|
||||
)
|
||||
|
||||
|
||||
# --- Unit Tests: Sparse patch application (CPU) ---
|
||||
|
||||
|
||||
class TestSparseNCCLPatchApplication:
|
||||
"""Test SparseNCCLWeightTransferEngine._apply_patch on a real param."""
|
||||
|
||||
def _make_engine(self, model):
|
||||
config = WeightTransferConfig(backend="sparse_nccl")
|
||||
return SparseNCCLWeightTransferEngine(
|
||||
config, create_mock_vllm_config(), torch.device("cpu"), model
|
||||
)
|
||||
|
||||
def _make_model(self, numel: int = 8):
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False)
|
||||
)
|
||||
|
||||
def get_parameter(name):
|
||||
assert name == "w"
|
||||
return model.w
|
||||
|
||||
model.get_parameter = get_parameter
|
||||
return model
|
||||
|
||||
def test_apply_patch_updates_only_selected_entries(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int32),
|
||||
values=torch.tensor([5.0, 7.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
expected = torch.zeros(8)
|
||||
expected[1] = 5.0
|
||||
expected[3] = 7.0
|
||||
assert torch.equal(model.w.data, expected)
|
||||
|
||||
def test_apply_patch_rejects_mismatched_lengths(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(ValueError, match="matching lengths"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1, 3], dtype=torch.int32),
|
||||
values=torch.tensor([5.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
|
||||
def test_apply_patch_rejects_non_int32_indices(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(ValueError, match="int32 indices"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1], dtype=torch.int64),
|
||||
values=torch.tensor([5.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
|
||||
def test_apply_patch_rejects_dtype_mismatch(self):
|
||||
model = self._make_model(8)
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(ValueError, match="does not match"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([5.0], dtype=torch.bfloat16),
|
||||
)
|
||||
)
|
||||
|
||||
def test_apply_patch_rejects_non_contiguous_param(self):
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w",
|
||||
torch.nn.Parameter(
|
||||
torch.arange(12, dtype=torch.float32).view(3, 4).t(),
|
||||
requires_grad=False,
|
||||
),
|
||||
)
|
||||
model.get_parameter = lambda name: model.w
|
||||
engine = self._make_engine(model)
|
||||
with pytest.raises(NotImplementedError, match="contiguous params"):
|
||||
engine._apply_patch(
|
||||
SparseWeightPatch(
|
||||
name="w",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
# --- Test receive_weights without init raises ---
|
||||
|
||||
|
||||
@@ -628,162 +532,135 @@ def test_nccl_weight_transfer_between_processes():
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_sparse_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
) -> bool:
|
||||
"""Trainer task that broadcasts sparse patches via the trainer engine.
|
||||
def test_sparse_nccl_checkpoint_chunks_to_ep_local_experts_cpu(monkeypatch):
|
||||
"""Replay global expert patches through two EP-local loaders on CPU."""
|
||||
|
||||
The worker task drives its own init/receive directly (it is not an RPC
|
||||
endpoint), so the engine gets a no-op control-plane client; the NCCL
|
||||
rendezvous and the patch broadcasts are the real thing.
|
||||
"""
|
||||
import torch
|
||||
|
||||
device = _set_ray_assigned_device()
|
||||
|
||||
from vllm.distributed.weight_transfer import WeightTransferTrainerFactory
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLTrainerInitInfo,
|
||||
SparseWeightPatch,
|
||||
)
|
||||
|
||||
class NoopClient:
|
||||
def init_weight_transfer_engine(self, init_info):
|
||||
pass
|
||||
class CaptureClient:
|
||||
def __init__(self):
|
||||
self.order: list[str] = []
|
||||
self.update_infos: list[dict] = []
|
||||
|
||||
def start_weight_update(self):
|
||||
pass
|
||||
self.order.append("start")
|
||||
|
||||
def update_weights(self, update_info):
|
||||
pass
|
||||
self.order.append("update")
|
||||
self.update_infos.append(update_info)
|
||||
|
||||
def finish_weight_update(self):
|
||||
pass
|
||||
self.order.append("finish")
|
||||
|
||||
patch = SparseWeightPatch(
|
||||
name="test.weight",
|
||||
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
|
||||
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
|
||||
full_shape=(10, 10),
|
||||
)
|
||||
engine = WeightTransferTrainerFactory.trainer_init(
|
||||
init_info=SparseNCCLTrainerInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
world_size=world_size,
|
||||
rank=0,
|
||||
),
|
||||
client=NoopClient(),
|
||||
)
|
||||
engine.send_weights([patch])
|
||||
torch.accelerator.synchronize()
|
||||
engine.shutdown()
|
||||
return True
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", lambda *_, **__: None)
|
||||
monkeypatch.setattr(torch.cuda, "is_available", lambda: False)
|
||||
monkeypatch.setattr(torch.accelerator, "device_index", lambda *_: nullcontext())
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def inference_receive_sparse_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
) -> dict:
|
||||
"""Inference task that receives sparse patches via the sparse engine."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
device = _set_ray_assigned_device()
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
|
||||
SparseNCCLWeightTransferEngine,
|
||||
SparseNCCLWeightTransferUpdateInfo,
|
||||
client = CaptureClient()
|
||||
sender = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
sender.model_update_group = MagicMock()
|
||||
sender.model_update_group.device = torch.device("cpu")
|
||||
wire_payloads = []
|
||||
sender.model_update_group.broadcast.side_effect = lambda tensor, **_: (
|
||||
wire_payloads.append(tensor.clone())
|
||||
)
|
||||
|
||||
config = WeightTransferConfig(backend="sparse_nccl")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
# Real module holding the target parameter the patch will modify.
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False)
|
||||
)
|
||||
model.get_parameter = lambda name: model.w
|
||||
|
||||
update_info = SparseNCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[30]],
|
||||
num_updates_list=[3],
|
||||
)
|
||||
|
||||
engine = SparseNCCLWeightTransferEngine(
|
||||
config, vllm_config, torch.device("cuda"), model
|
||||
)
|
||||
from vllm.distributed.weight_transfer.nccl_common import (
|
||||
NCCLWeightTransferInitInfo,
|
||||
)
|
||||
|
||||
engine.init_transfer_engine(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1,
|
||||
world_size=world_size,
|
||||
expert_names = [
|
||||
"model.layers.0.mlp.experts.0.gate_proj.weight",
|
||||
"model.layers.0.mlp.experts.1.gate_proj.weight",
|
||||
]
|
||||
client.start_weight_update()
|
||||
for name, index, value in zip(
|
||||
expert_names,
|
||||
(0, 3),
|
||||
(5.0, 7.0),
|
||||
strict=True,
|
||||
):
|
||||
sender.send_weight_chunk(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name=name,
|
||||
indices=torch.tensor([index], dtype=torch.int32),
|
||||
values=torch.tensor([value]),
|
||||
full_shape=(2, 2),
|
||||
)
|
||||
]
|
||||
)
|
||||
client.finish_weight_update()
|
||||
|
||||
assert client.order == ["start", "update", "update", "finish"]
|
||||
assert [info["names"] for info in client.update_infos] == [
|
||||
[expert_names[0]],
|
||||
[expert_names[1]],
|
||||
]
|
||||
assert [info["shapes"] for info in client.update_infos] == [
|
||||
[[2, 2]],
|
||||
[[2, 2]],
|
||||
]
|
||||
assert [info["num_updates_list"] for info in client.update_infos] == [
|
||||
[1],
|
||||
[1],
|
||||
]
|
||||
|
||||
expected_gates = (
|
||||
[[5.0, -1.0], [-1.0, -1.0]],
|
||||
[[-1.0, -1.0], [-1.0, 7.0]],
|
||||
)
|
||||
engine.receive_weights(update_info)
|
||||
torch.accelerator.synchronize()
|
||||
for ep_rank, expected_gate in enumerate(expected_gates):
|
||||
model = torch.nn.Module()
|
||||
model.register_parameter(
|
||||
"w13_weight",
|
||||
torch.nn.Parameter(torch.full((1, 4, 2), -1.0), requires_grad=False),
|
||||
)
|
||||
local_name = expert_names[ep_rank]
|
||||
seen_names: list[str] = []
|
||||
loaded_names: set[str] = set()
|
||||
copy_count = [0]
|
||||
|
||||
expected = torch.zeros(30, dtype=torch.float32, device=device)
|
||||
expected[[1, 7, 25]] = torch.tensor(
|
||||
[10.0, 20.0, 30.0], dtype=torch.float32, device=device
|
||||
)
|
||||
success = torch.equal(model.w.data, expected)
|
||||
engine.shutdown()
|
||||
return {
|
||||
"success": success,
|
||||
"selected_values": model.w.data[[1, 7, 25]].cpu().tolist(),
|
||||
}
|
||||
def load_weights(
|
||||
weights,
|
||||
local_name=local_name,
|
||||
target=model,
|
||||
seen=seen_names,
|
||||
loaded=loaded_names,
|
||||
count=copy_count,
|
||||
):
|
||||
for name, checkpoint_weight in weights:
|
||||
seen.append(name)
|
||||
if name != local_name:
|
||||
continue
|
||||
target.w13_weight.data[0, :2].copy_(checkpoint_weight)
|
||||
loaded.add(name)
|
||||
count[0] += 1
|
||||
return loaded
|
||||
|
||||
model.load_weights = load_weights
|
||||
receiver = SparseNCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="sparse_nccl"),
|
||||
create_mock_vllm_config(rank=ep_rank, world_size=2),
|
||||
torch.device("cpu"),
|
||||
model,
|
||||
)
|
||||
receiver.model_update_group = MagicMock()
|
||||
receiver.model_update_group.device = torch.device("cpu")
|
||||
payloads = iter(wire_payloads)
|
||||
receiver.model_update_group.broadcast.side_effect = (
|
||||
lambda tensor, payloads=payloads, **_: tensor.copy_(next(payloads))
|
||||
)
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL sparse weight transfer test.",
|
||||
)
|
||||
def test_nccl_sparse_weight_transfer_between_processes():
|
||||
"""Test NCCL sparse weight transfer from trainer to inference process."""
|
||||
_init_ray_for_weight_transfer()
|
||||
receiver.start_weight_update()
|
||||
for update_info in client.update_infos:
|
||||
receiver.receive_weights(SparseNCCLWeightTransferUpdateInfo(**update_info))
|
||||
receiver.finish_weight_update()
|
||||
|
||||
master_address = "127.0.0.1"
|
||||
master_port = get_open_port()
|
||||
world_size = 2
|
||||
assert seen_names == expert_names
|
||||
assert loaded_names == {local_name}
|
||||
assert copy_count == [1]
|
||||
assert model.w13_weight[0, :2].tolist() == expected_gate
|
||||
assert model.w13_weight[0, 2:].eq(-1).all()
|
||||
foreign_value = 7.0 if ep_rank == 0 else 5.0
|
||||
assert not model.w13_weight.eq(foreign_value).any()
|
||||
receiver.shutdown()
|
||||
|
||||
inference_future = inference_receive_sparse_tensor.remote(
|
||||
master_address, master_port, world_size
|
||||
)
|
||||
trainer_future = trainer_broadcast_sparse_tensor.remote(
|
||||
master_address, master_port, world_size
|
||||
)
|
||||
|
||||
trainer_result, result = ray.get([trainer_future, inference_future])
|
||||
|
||||
assert trainer_result, "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
"Sparse weight transfer failed. "
|
||||
f"Received selected values: {result['selected_values']}"
|
||||
)
|
||||
sender.shutdown()
|
||||
|
||||
|
||||
# --- Unit Tests: IPCWeightTransferUpdateInfo Validation ---
|
||||
@@ -1995,11 +1872,11 @@ def test_sparse_nccl_trainer_init_ships_worker_init_info(monkeypatch):
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
|
||||
"""send_weights takes the round's patches and ships per-patch metadata
|
||||
(names / shapes / num_updates_list) + broadcasts indices + values each."""
|
||||
"""One-shot send drives start, update, and finish in order."""
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
engine.model_update_group.device = torch.device("cpu")
|
||||
# The group is a mock, so the stream is just a handle it is handed (and a
|
||||
# handle _post_send_sync can synchronize).
|
||||
monkeypatch.setattr(torch.cuda, "current_stream", MagicMock())
|
||||
@@ -2013,6 +1890,7 @@ def test_sparse_nccl_trainer_send_weights_drives_client_in_order(monkeypatch):
|
||||
assert client.last_update_info["num_updates_list"] == [2]
|
||||
# One broadcast for indices + one for values per patch.
|
||||
assert engine.model_update_group.broadcast.call_count == 2
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
||||
@@ -2024,6 +1902,8 @@ def test_sparse_nccl_trainer_send_weights_empty_round_is_noop():
|
||||
|
||||
engine.send_weights([])
|
||||
engine.send_weights() # no argument is also a no-op round
|
||||
engine.send_weight_chunk([])
|
||||
engine.send_weight_chunk()
|
||||
|
||||
assert client.order == []
|
||||
|
||||
@@ -2033,6 +1913,7 @@ def test_sparse_nccl_trainer_send_weights_requires_full_shape():
|
||||
patch.full_shape = None
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=RecordingClient())
|
||||
engine.model_update_group = MagicMock()
|
||||
engine.model_update_group.device = torch.device("cpu")
|
||||
|
||||
with pytest.raises(ValueError, match="full_shape"):
|
||||
engine.send_weights([patch])
|
||||
@@ -2054,6 +1935,7 @@ def test_sparse_nccl_trainer_validates_patch_before_any_rpc():
|
||||
client = RecordingClient()
|
||||
engine = SparseNCCLTrainerWeightTransferEngine(client=client)
|
||||
engine.model_update_group = MagicMock()
|
||||
engine.model_update_group.device = torch.device("cpu")
|
||||
|
||||
mismatched = SparseWeightPatch(
|
||||
name="w",
|
||||
@@ -2092,4 +1974,5 @@ def test_sparse_nccl_trainer_non_sender_skips_client():
|
||||
assert engine.model_update_group is None
|
||||
assert isinstance(engine, SparseNCCLTrainerWeightTransferEngine)
|
||||
engine.send_weights([_sparse_patch()])
|
||||
engine.send_weight_chunk([_sparse_patch()])
|
||||
assert client.order == []
|
||||
|
||||
Reference in New Issue
Block a user