Weight sync refactor + move sparse nccl engine (#44353)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-07-01 01:25:19 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent f651a8a9a4
commit 77a9c5ae28
32 changed files with 951 additions and 1044 deletions
+266 -253
View File
@@ -18,7 +18,6 @@ from torch.multiprocessing.reductions import reduce_tensor
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.ipc_engine import (
IPCWeightTransferEngine,
IPCWeightTransferInitInfo,
@@ -29,6 +28,11 @@ from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
SparseWeightPatch,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
@@ -80,6 +84,18 @@ def create_mock_parallel_config(
return config
def create_mock_vllm_config(
rank: int = 0,
world_size: int = 1,
dp_rank: int = 0,
) -> MagicMock:
"""Create a mock VllmConfig exposing parallel_config and model_config."""
vllm_config = MagicMock()
vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank)
vllm_config.model_config = MagicMock()
return vllm_config
# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation ---
@@ -87,7 +103,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
"""Test NCCLWeightTransferUpdateInfo dataclass validation."""
def test_valid_update_info(self):
"""Test creating valid NCCLWeightTransferUpdateInfo."""
info = NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
@@ -98,7 +113,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
assert info.shapes == [[10, 10], [10]]
def test_mismatched_dtype_names_raises(self):
"""Test that mismatched dtype_names length raises ValueError."""
with pytest.raises(ValueError, match="dtype_names"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
@@ -107,7 +121,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
def test_mismatched_shapes_raises(self):
"""Test that mismatched shapes length raises ValueError."""
with pytest.raises(ValueError, match="shapes"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
@@ -116,73 +129,59 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
def test_empty_lists_valid(self):
"""Test that empty lists are valid."""
info = NCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
)
info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[])
assert len(info.names) == 0
# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation ---
class TestSparseNCCLWeightTransferUpdateInfoValidation:
"""Test SparseNCCLWeightTransferUpdateInfo dataclass validation."""
def test_valid_sparse_update_info(self):
"""Test creating valid sparse NCCL update info."""
info = NCCLWeightTransferUpdateInfo(
info = SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "bfloat16"],
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
update_kind="sparse_flat",
)
assert info.update_kind == "sparse_flat"
assert info.num_updates_list == [4, 2]
def test_sparse_update_requires_num_updates_list(self):
with pytest.raises(ValueError, match="`num_updates_list` is required"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
def test_mismatched_dtype_names_raises(self):
with pytest.raises(ValueError, match="dtype_names"):
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32"],
shapes=[[10, 10]],
update_kind="sparse_flat",
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
)
def test_sparse_update_rejects_empty_num_updates_list(self):
def test_rejects_empty_num_updates_list(self):
with pytest.raises(ValueError, match="cannot be empty"):
NCCLWeightTransferUpdateInfo(
SparseNCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
num_updates_list=[],
update_kind="sparse_flat",
)
def test_sparse_update_rejects_packed(self):
with pytest.raises(ValueError, match="cannot be combined with `packed=True`"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
update_kind="sparse_flat",
packed=True,
)
def test_sparse_update_rejects_mismatched_num_updates(self):
def test_rejects_mismatched_num_updates(self):
with pytest.raises(ValueError, match="`num_updates_list`"):
NCCLWeightTransferUpdateInfo(
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
shapes=[[10, 10], [10]],
num_updates_list=[3],
update_kind="sparse_flat",
)
def test_dense_update_rejects_sparse_metadata(self):
with pytest.raises(ValueError, match="Sparse metadata"):
NCCLWeightTransferUpdateInfo(
def test_rejects_negative_num_updates(self):
with pytest.raises(ValueError, match="non-negative"):
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
num_updates_list=[-1],
)
@@ -192,14 +191,17 @@ class TestNCCLWeightTransferUpdateInfoValidation:
class TestNCCLEngineParsing:
"""Test NCCLWeightTransferEngine parsing methods."""
def test_parse_init_info_valid(self):
"""Test parsing valid init info dict."""
def _make_engine(self):
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
return NCCLWeightTransferEngine(
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_parse_init_info_valid(self):
engine = self._make_engine()
init_info = engine.parse_init_info(
{
"master_address": "127.0.0.1",
@@ -208,7 +210,6 @@ class TestNCCLEngineParsing:
"world_size": 3,
}
)
assert isinstance(init_info, NCCLWeightTransferInitInfo)
assert init_info.master_address == "127.0.0.1"
assert init_info.master_port == 12345
@@ -216,29 +217,12 @@ class TestNCCLEngineParsing:
assert init_info.world_size == 3
def test_parse_init_info_missing_field_raises(self):
"""Test parsing init info with missing required field."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
with pytest.raises(ValueError, match="Invalid init_info"):
engine.parse_init_info(
{
"master_address": "127.0.0.1",
# Missing master_port, rank_offset, world_size
}
)
engine.parse_init_info({"master_address": "127.0.0.1"})
def test_parse_update_info_valid(self):
"""Test parsing valid update info dict."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
update_info = engine.parse_update_info(
{
"names": ["w1", "w2"],
@@ -246,7 +230,6 @@ class TestNCCLEngineParsing:
"shapes": [[100, 100], [50]],
}
)
assert isinstance(update_info, NCCLWeightTransferUpdateInfo)
assert update_info.names == ["w1", "w2"]
assert update_info.dtype_names == ["float32", "bfloat16"]
@@ -260,40 +243,140 @@ class TestEngineRegistry:
"""Test weight transfer engine registry."""
def test_create_engine_nccl(self):
"""Test factory creates NCCL engine."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, NCCLWeightTransferEngine)
def test_create_engine_ipc(self):
"""Test factory creates IPC engine."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, IPCWeightTransferEngine)
def test_create_engine_sparse_nccl(self):
config = WeightTransferConfig(backend="sparse_nccl")
engine = WeightTransferEngineFactory.create_engine(
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, SparseNCCLWeightTransferEngine)
def test_create_engine_invalid_backend(self):
"""Test factory raises for invalid backend."""
config = WeightTransferConfig(backend="invalid")
parallel_config = create_mock_parallel_config()
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_register_duplicate_raises(self):
"""Test registering duplicate engine name raises."""
with pytest.raises(ValueError, match="already registered"):
WeightTransferEngineFactory.register_engine(
"nccl", NCCLWeightTransferEngine
)
# --- Unit Tests: Sparse patch application (CPU) ---
class TestSparseNCCLPatchApplication:
"""Test SparseNCCLWeightTransferEngine._apply_patch on a real param."""
def _make_engine(self, model):
config = WeightTransferConfig(backend="sparse_nccl")
return SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), "cpu", model
)
def _make_model(self, numel: int = 8):
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False)
)
def get_parameter(name):
assert name == "w"
return model.w
model.get_parameter = get_parameter
return model
def test_apply_patch_updates_only_selected_entries(self):
model = self._make_model(8)
engine = self._make_engine(model)
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0, 7.0], dtype=torch.float32),
)
)
expected = torch.zeros(8)
expected[1] = 5.0
expected[3] = 7.0
assert torch.equal(model.w.data, expected)
def test_apply_patch_rejects_mismatched_lengths(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="matching lengths"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_non_int32_indices(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="int32 indices"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int64),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_dtype_mismatch(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="does not match"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.bfloat16),
)
)
def test_apply_patch_rejects_non_contiguous_param(self):
model = torch.nn.Module()
model.register_parameter(
"w",
torch.nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t(),
requires_grad=False,
),
)
model.get_parameter = lambda name: model.w
engine = self._make_engine(model)
with pytest.raises(NotImplementedError, match="contiguous params"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
)
# --- Test receive_weights without init raises ---
@@ -303,42 +386,37 @@ def test_nccl_receive_weights_without_init_raises():
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
update_info = NCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
names=["w"], dtype_names=["float32"], shapes=[[10]]
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
def test_nccl_receive_sparse_weights_without_init_raises():
def test_sparse_nccl_receive_weights_without_init_raises():
"""Test that sparse receive raises if init_transfer_engine wasn't called."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config = WeightTransferConfig(backend="sparse_nccl")
engine = SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
update_info = NCCLWeightTransferUpdateInfo(
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
num_updates_list=[2],
update_kind="sparse_flat",
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_sparse_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
@@ -387,6 +465,7 @@ def inference_receive_tensor(
tensor_dtype: str,
) -> dict:
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
import contextlib
from unittest.mock import MagicMock
import torch
@@ -401,17 +480,32 @@ def inference_receive_tensor(
NCCLWeightTransferUpdateInfo,
)
# Create engine with mock parallel config
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
recorder = Recorder()
engine = NCCLWeightTransferEngine(config, vllm_config, "cuda", recorder)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (joins as rank 1)
init_info = NCCLWeightTransferInitInfo(
@@ -422,20 +516,12 @@ def inference_receive_tensor(
)
engine.init_transfer_engine(init_info)
# Receive weights with a no-op load_weights that captures the tensor
received_tensors = []
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
for name, tensor in weights:
# Clone tensor to keep it after engine cleans up
received_tensors.append((name, tensor.clone()))
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=[tensor_dtype],
shapes=[tensor_shape],
)
engine.receive_weights(update_info, noop_load_weights)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
# Verify we received the tensor
@@ -443,11 +529,10 @@ def inference_receive_tensor(
received_shape = None
received_sum = None
if len(received_tensors) == 1:
name, tensor = received_tensors[0]
if len(recorder.received) == 1:
name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
# Check shape matches and values are all 1s (trainer sends ones)
if received_shape == tensor_shape:
expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item()
if abs(received_sum - expected_sum) < 0.01:
@@ -478,11 +563,9 @@ def test_nccl_weight_transfer_between_processes():
master_port = get_open_port()
world_size = 2 # 1 trainer + 1 inference worker
# Tensor to transfer: 100x100 ones
tensor_shape = [100, 100]
tensor_dtype = "float32"
# Start both tasks concurrently - Ray assigns GPUs automatically
inference_future = inference_receive_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
@@ -490,7 +573,6 @@ def test_nccl_weight_transfer_between_processes():
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
# Wait for both to complete
trainer_result, result = ray.get([trainer_future, inference_future])
assert trainer_result, "Trainer should complete successfully"
@@ -514,10 +596,12 @@ def trainer_broadcast_sparse_tensor(
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseWeightPatch,
)
pg = StatelessProcessGroup.create(
@@ -533,7 +617,7 @@ def trainer_broadcast_sparse_tensor(
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
)
NCCLWeightTransferEngine.trainer_send_sparse_weights(
SparseNCCLWeightTransferEngine.trainer_send_weights(
iter([patch]),
NCCLTrainerSendWeightsArgs(group=comm),
)
@@ -547,7 +631,7 @@ def inference_receive_sparse_tensor(
master_port: int,
world_size: int,
) -> dict:
"""Inference task that receives sparse patches via NCCLWeightTransferEngine."""
"""Inference task that receives sparse patches via the sparse engine."""
from unittest.mock import MagicMock
import torch
@@ -556,22 +640,40 @@ def inference_receive_sparse_tensor(
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
)
config = WeightTransferConfig(backend="nccl")
config = WeightTransferConfig(backend="sparse_nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
# Real module holding the target parameter the patch will modify.
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False)
)
model.get_parameter = lambda name: model.w
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
)
engine = SparseNCCLWeightTransferEngine(config, vllm_config, "cuda", model)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
)
engine.init_transfer_engine(
NCCLWeightTransferInitInfo(
master_address=master_address,
@@ -580,32 +682,18 @@ def inference_receive_sparse_tensor(
world_size=world_size,
)
)
target = torch.zeros(30, dtype=torch.float32, device=device)
def apply_sparse_patches(patches: list[SparseWeightPatch]):
for patch in patches:
target.index_copy_(0, patch.indices.to(torch.long), patch.values)
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
update_kind="sparse_flat",
)
engine.receive_sparse_weights(update_info, apply_sparse_patches)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
expected = torch.zeros(30, dtype=torch.float32, device=device)
expected[[1, 7, 25]] = torch.tensor(
[10.0, 20.0, 30.0], dtype=torch.float32, device=device
)
success = torch.equal(target, expected)
success = torch.equal(model.w.data, expected)
engine.shutdown()
return {
"success": success,
"selected_values": target[[1, 7, 25]].cpu().tolist(),
"selected_values": model.w.data[[1, 7, 25]].cpu().tolist(),
}
@@ -644,11 +732,9 @@ class TestIPCWeightTransferUpdateInfoValidation:
"""Test IPCWeightTransferUpdateInfo dataclass validation."""
def test_valid_update_info(self):
"""Test creating valid IPCWeightTransferUpdateInfo."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Create a dummy tensor and IPC handle
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
@@ -666,7 +752,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
assert len(info.ipc_handles) == 1
def test_mismatched_dtype_names_raises(self):
"""Test that mismatched dtype_names length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -684,7 +769,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_mismatched_shapes_raises(self):
"""Test that mismatched shapes length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -702,7 +786,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_mismatched_ipc_handles_raises(self):
"""Test that mismatched ipc_handles length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -719,44 +802,7 @@ class TestIPCWeightTransferUpdateInfoValidation:
ipc_handles=ipc_handles,
)
def test_sparse_update_kind_rejected(self):
"""Test that IPC backend rejects sparse update metadata."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
dummy_tensor = torch.ones(10, 10, device="cuda:0")
ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
ipc_handles = [{gpu_uuid: ipc_handle}]
with pytest.raises(NotImplementedError, match="dense updates"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[1],
ipc_handles=ipc_handles,
update_kind="sparse_flat",
)
def test_sparse_methods_not_supported(self):
"""Test that IPC engine inherits sparse rejection from the base class."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.receive_sparse_weights(MagicMock(), lambda _: None)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.trainer_send_sparse_weights(
iter([]),
{"mode": "http", "url": "http://localhost:8000"},
)
def test_valid_update_info_from_pickled(self, monkeypatch):
"""Test creating IPCWeightTransferUpdateInfo from pickled handles."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -779,7 +825,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
assert info.ipc_handles_pickled is None
def test_pickled_requires_insecure_serialization_flag(self, monkeypatch):
"""Test that pickled handles are rejected unless env flag is enabled."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")
with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"):
@@ -791,7 +836,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -812,7 +856,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_neither_handles_nor_pickled_raises(self):
"""Test that providing neither ipc_handles nor ipc_handles_pickled raises."""
with pytest.raises(ValueError, match="must be provided"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
@@ -821,7 +864,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_empty_lists_valid(self):
"""Test that empty lists are valid."""
info = IPCWeightTransferUpdateInfo(
names=[],
dtype_names=[],
@@ -837,18 +879,21 @@ class TestIPCWeightTransferUpdateInfoValidation:
class TestIPCEngineParsing:
"""Test IPCWeightTransferEngine parsing methods."""
def _make_engine(self):
config = WeightTransferConfig(backend="ipc")
return IPCWeightTransferEngine(
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_parse_update_info_valid(self):
"""Test parsing valid update info dict."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
# Create dummy IPC handles
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
dummy_tensor2 = torch.ones(50, device="cuda:0")
_, ipc_args1 = reduce_tensor(dummy_tensor1)
@@ -872,17 +917,12 @@ class TestIPCEngineParsing:
assert len(update_info.ipc_handles) == 2
def test_parse_update_info_pickled(self, monkeypatch):
"""Test parsing update info with pickled IPC handles (HTTP path)."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
dummy_tensor2 = torch.ones(50, device="cuda:0")
@@ -909,12 +949,7 @@ class TestIPCEngineParsing:
assert gpu_uuid in update_info.ipc_handles[1]
def test_parse_update_info_ignores_none_pickled_handles(self):
"""Test Ray/asdict payloads with a null pickled field use ipc_handles."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
ipc_handles = [{"gpu-uuid": ("ipc-args",)}]
update_info = engine.parse_update_info(
@@ -931,15 +966,10 @@ class TestIPCEngineParsing:
assert update_info.ipc_handles == ipc_handles
def test_parse_update_info_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
@@ -981,9 +1011,6 @@ class TrainerActor:
self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device)
self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer
# Create IPC handle (tensor must stay alive for IPC to work)
# reduce_tensor returns (rebuild_func, args); we only send args
# since the receiver imports rebuild_cuda_tensor directly.
_, ipc_args = reduce_tensor(self.tensor)
gpu_uuid = get_physical_gpu_id(device.index)
@@ -1007,6 +1034,7 @@ def inference_receive_ipc_tensor(
mode: str = "ray",
) -> dict:
"""Inference task that receives tensor via IPCWeightTransferEngine."""
import contextlib
import os
# Worker-side: ipc_handles_pickled is deserialized via pickle.
@@ -1025,30 +1053,36 @@ def inference_receive_ipc_tensor(
IPCWeightTransferEngine,
)
# Create engine with mock parallel config
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="ipc")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
recorder = Recorder()
engine = IPCWeightTransferEngine(config, vllm_config, "cuda", recorder)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (no-op for IPC)
init_info = IPCWeightTransferInitInfo()
engine.init_transfer_engine(init_info)
# Receive weights with a no-op load_weights that captures the tensor
received_tensors = []
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
for name, tensor in weights:
# Clone tensor to keep it after engine cleans up
received_tensors.append((name, tensor.clone()))
ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}]
if mode == "ray":
@@ -1059,7 +1093,6 @@ def inference_receive_ipc_tensor(
"ipc_handles": ipc_handles,
}
elif mode == "http":
# Simulate HTTP transport: pickle + base64 encode handles
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
update_dict = {
"names": ["test.weight"],
@@ -1071,19 +1104,17 @@ def inference_receive_ipc_tensor(
raise ValueError(f"Unknown mode: {mode}")
update_info = engine.parse_update_info(update_dict)
engine.receive_weights(update_info, noop_load_weights)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
# Verify we received the tensor
success = False
received_shape = None
received_sum = None
if len(received_tensors) == 1:
name, tensor = received_tensors[0]
if len(recorder.received) == 1:
name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
# Check shape matches and values are all 42s (trainer sends 42s)
if received_shape == ipc_handle_dict["shape"]:
expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item()
if abs(received_sum - expected_sum) < 0.01:
@@ -1104,23 +1135,12 @@ def inference_receive_ipc_tensor(
)
@pytest.mark.parametrize("mode", ["ray", "http"])
def test_ipc_weight_transfer_between_processes(mode: str):
"""Test IPC weight transfer from trainer to inference process using Ray.
Parametrized over transport modes:
- 'ray': ipc_handles passed directly.
- 'http': ipc_handles pickled + base64-encoded, deserialized in
parse_update_info before constructing the dataclass.
IPC requires same-GPU access, so we use a placement group to co-locate
the trainer actor and inference task on the same GPU.
"""
"""Test IPC weight transfer from trainer to inference process using Ray."""
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
_init_ray_for_weight_transfer()
# Create a placement group to ensure both processes are on the same GPU
# Use fractional GPUs so both tasks can share the same GPU bundle
pg = placement_group([{"GPU": 1, "CPU": 2}])
ray.get(pg.ready())
@@ -1129,20 +1149,15 @@ def test_ipc_weight_transfer_between_processes(mode: str):
placement_group_capture_child_tasks=True,
)
# Tensor to transfer: 100x100 filled with 42s
tensor_shape = [100, 100]
tensor_dtype = "float32"
# Create trainer actor that holds the tensor and IPC handle (stays alive)
trainer_actor = TrainerActor.options( # type: ignore[attr-defined]
scheduling_strategy=scheduling_strategy
).remote(tensor_shape, tensor_dtype)
# Get IPC handle dict (tensor stays alive in trainer actor)
ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote())
# Receive tensor in inference process using IPC handles (on same GPU)
# Trainer actor stays alive during this operation
inference_result = ray.get(
inference_receive_ipc_tensor.options(
scheduling_strategy=scheduling_strategy
@@ -1162,12 +1177,10 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
# Create IPC handle with wrong GPU UUID
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
wrong_uuid = "wrong-uuid-12345"
@@ -1181,4 +1194,4 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
)
with pytest.raises(ValueError, match="IPC handle not found"):
engine.receive_weights(update_info, lambda x: None)
engine.receive_weights(update_info)