Weight sync refactor + move sparse nccl engine (#44353)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-07-01 01:25:19 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent f651a8a9a4
commit 77a9c5ae28
32 changed files with 951 additions and 1044 deletions
+266 -253
View File
@@ -18,7 +18,6 @@ from torch.multiprocessing.reductions import reduce_tensor
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.ipc_engine import (
IPCWeightTransferEngine,
IPCWeightTransferInitInfo,
@@ -29,6 +28,11 @@ from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
SparseWeightPatch,
)
from vllm.platforms import current_platform
from vllm.utils.network_utils import get_open_port
@@ -80,6 +84,18 @@ def create_mock_parallel_config(
return config
def create_mock_vllm_config(
rank: int = 0,
world_size: int = 1,
dp_rank: int = 0,
) -> MagicMock:
"""Create a mock VllmConfig exposing parallel_config and model_config."""
vllm_config = MagicMock()
vllm_config.parallel_config = create_mock_parallel_config(rank, world_size, dp_rank)
vllm_config.model_config = MagicMock()
return vllm_config
# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation ---
@@ -87,7 +103,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
"""Test NCCLWeightTransferUpdateInfo dataclass validation."""
def test_valid_update_info(self):
"""Test creating valid NCCLWeightTransferUpdateInfo."""
info = NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
@@ -98,7 +113,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
assert info.shapes == [[10, 10], [10]]
def test_mismatched_dtype_names_raises(self):
"""Test that mismatched dtype_names length raises ValueError."""
with pytest.raises(ValueError, match="dtype_names"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
@@ -107,7 +121,6 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
def test_mismatched_shapes_raises(self):
"""Test that mismatched shapes length raises ValueError."""
with pytest.raises(ValueError, match="shapes"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
@@ -116,73 +129,59 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
def test_empty_lists_valid(self):
"""Test that empty lists are valid."""
info = NCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
)
info = NCCLWeightTransferUpdateInfo(names=[], dtype_names=[], shapes=[])
assert len(info.names) == 0
# --- Unit Tests: SparseNCCLWeightTransferUpdateInfo Validation ---
class TestSparseNCCLWeightTransferUpdateInfoValidation:
"""Test SparseNCCLWeightTransferUpdateInfo dataclass validation."""
def test_valid_sparse_update_info(self):
"""Test creating valid sparse NCCL update info."""
info = NCCLWeightTransferUpdateInfo(
info = SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "bfloat16"],
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
update_kind="sparse_flat",
)
assert info.update_kind == "sparse_flat"
assert info.num_updates_list == [4, 2]
def test_sparse_update_requires_num_updates_list(self):
with pytest.raises(ValueError, match="`num_updates_list` is required"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
def test_mismatched_dtype_names_raises(self):
with pytest.raises(ValueError, match="dtype_names"):
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32"],
shapes=[[10, 10]],
update_kind="sparse_flat",
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
)
def test_sparse_update_rejects_empty_num_updates_list(self):
def test_rejects_empty_num_updates_list(self):
with pytest.raises(ValueError, match="cannot be empty"):
NCCLWeightTransferUpdateInfo(
SparseNCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
num_updates_list=[],
update_kind="sparse_flat",
)
def test_sparse_update_rejects_packed(self):
with pytest.raises(ValueError, match="cannot be combined with `packed=True`"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
update_kind="sparse_flat",
packed=True,
)
def test_sparse_update_rejects_mismatched_num_updates(self):
def test_rejects_mismatched_num_updates(self):
with pytest.raises(ValueError, match="`num_updates_list`"):
NCCLWeightTransferUpdateInfo(
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
shapes=[[10, 10], [10]],
num_updates_list=[3],
update_kind="sparse_flat",
)
def test_dense_update_rejects_sparse_metadata(self):
with pytest.raises(ValueError, match="Sparse metadata"):
NCCLWeightTransferUpdateInfo(
def test_rejects_negative_num_updates(self):
with pytest.raises(ValueError, match="non-negative"):
SparseNCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
num_updates_list=[-1],
)
@@ -192,14 +191,17 @@ class TestNCCLWeightTransferUpdateInfoValidation:
class TestNCCLEngineParsing:
"""Test NCCLWeightTransferEngine parsing methods."""
def test_parse_init_info_valid(self):
"""Test parsing valid init info dict."""
def _make_engine(self):
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
return NCCLWeightTransferEngine(
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_parse_init_info_valid(self):
engine = self._make_engine()
init_info = engine.parse_init_info(
{
"master_address": "127.0.0.1",
@@ -208,7 +210,6 @@ class TestNCCLEngineParsing:
"world_size": 3,
}
)
assert isinstance(init_info, NCCLWeightTransferInitInfo)
assert init_info.master_address == "127.0.0.1"
assert init_info.master_port == 12345
@@ -216,29 +217,12 @@ class TestNCCLEngineParsing:
assert init_info.world_size == 3
def test_parse_init_info_missing_field_raises(self):
"""Test parsing init info with missing required field."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
with pytest.raises(ValueError, match="Invalid init_info"):
engine.parse_init_info(
{
"master_address": "127.0.0.1",
# Missing master_port, rank_offset, world_size
}
)
engine.parse_init_info({"master_address": "127.0.0.1"})
def test_parse_update_info_valid(self):
"""Test parsing valid update info dict."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
update_info = engine.parse_update_info(
{
"names": ["w1", "w2"],
@@ -246,7 +230,6 @@ class TestNCCLEngineParsing:
"shapes": [[100, 100], [50]],
}
)
assert isinstance(update_info, NCCLWeightTransferUpdateInfo)
assert update_info.names == ["w1", "w2"]
assert update_info.dtype_names == ["float32", "bfloat16"]
@@ -260,40 +243,140 @@ class TestEngineRegistry:
"""Test weight transfer engine registry."""
def test_create_engine_nccl(self):
"""Test factory creates NCCL engine."""
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, NCCLWeightTransferEngine)
def test_create_engine_ipc(self):
"""Test factory creates IPC engine."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, IPCWeightTransferEngine)
def test_create_engine_sparse_nccl(self):
config = WeightTransferConfig(backend="sparse_nccl")
engine = WeightTransferEngineFactory.create_engine(
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
assert isinstance(engine, SparseNCCLWeightTransferEngine)
def test_create_engine_invalid_backend(self):
"""Test factory raises for invalid backend."""
config = WeightTransferConfig(backend="invalid")
parallel_config = create_mock_parallel_config()
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
WeightTransferEngineFactory.create_engine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_register_duplicate_raises(self):
"""Test registering duplicate engine name raises."""
with pytest.raises(ValueError, match="already registered"):
WeightTransferEngineFactory.register_engine(
"nccl", NCCLWeightTransferEngine
)
# --- Unit Tests: Sparse patch application (CPU) ---
class TestSparseNCCLPatchApplication:
"""Test SparseNCCLWeightTransferEngine._apply_patch on a real param."""
def _make_engine(self, model):
config = WeightTransferConfig(backend="sparse_nccl")
return SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), "cpu", model
)
def _make_model(self, numel: int = 8):
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(numel), requires_grad=False)
)
def get_parameter(name):
assert name == "w"
return model.w
model.get_parameter = get_parameter
return model
def test_apply_patch_updates_only_selected_entries(self):
model = self._make_model(8)
engine = self._make_engine(model)
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0, 7.0], dtype=torch.float32),
)
)
expected = torch.zeros(8)
expected[1] = 5.0
expected[3] = 7.0
assert torch.equal(model.w.data, expected)
def test_apply_patch_rejects_mismatched_lengths(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="matching lengths"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1, 3], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_non_int32_indices(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="int32 indices"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int64),
values=torch.tensor([5.0], dtype=torch.float32),
)
)
def test_apply_patch_rejects_dtype_mismatch(self):
model = self._make_model(8)
engine = self._make_engine(model)
with pytest.raises(ValueError, match="does not match"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([5.0], dtype=torch.bfloat16),
)
)
def test_apply_patch_rejects_non_contiguous_param(self):
model = torch.nn.Module()
model.register_parameter(
"w",
torch.nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t(),
requires_grad=False,
),
)
model.get_parameter = lambda name: model.w
engine = self._make_engine(model)
with pytest.raises(NotImplementedError, match="contiguous params"):
engine._apply_patch(
SparseWeightPatch(
name="w",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
)
# --- Test receive_weights without init raises ---
@@ -303,42 +386,37 @@ def test_nccl_receive_weights_without_init_raises():
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
update_info = NCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
names=["w"], dtype_names=["float32"], shapes=[[10]]
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
def test_nccl_receive_sparse_weights_without_init_raises():
def test_sparse_nccl_receive_weights_without_init_raises():
"""Test that sparse receive raises if init_transfer_engine wasn't called."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config = WeightTransferConfig(backend="sparse_nccl")
engine = SparseNCCLWeightTransferEngine(
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
update_info = NCCLWeightTransferUpdateInfo(
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
num_updates_list=[2],
update_kind="sparse_flat",
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_sparse_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
@@ -387,6 +465,7 @@ def inference_receive_tensor(
tensor_dtype: str,
) -> dict:
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
import contextlib
from unittest.mock import MagicMock
import torch
@@ -401,17 +480,32 @@ def inference_receive_tensor(
NCCLWeightTransferUpdateInfo,
)
# Create engine with mock parallel config
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
recorder = Recorder()
engine = NCCLWeightTransferEngine(config, vllm_config, "cuda", recorder)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (joins as rank 1)
init_info = NCCLWeightTransferInitInfo(
@@ -422,20 +516,12 @@ def inference_receive_tensor(
)
engine.init_transfer_engine(init_info)
# Receive weights with a no-op load_weights that captures the tensor
received_tensors = []
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
for name, tensor in weights:
# Clone tensor to keep it after engine cleans up
received_tensors.append((name, tensor.clone()))
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=[tensor_dtype],
shapes=[tensor_shape],
)
engine.receive_weights(update_info, noop_load_weights)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
# Verify we received the tensor
@@ -443,11 +529,10 @@ def inference_receive_tensor(
received_shape = None
received_sum = None
if len(received_tensors) == 1:
name, tensor = received_tensors[0]
if len(recorder.received) == 1:
name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
# Check shape matches and values are all 1s (trainer sends ones)
if received_shape == tensor_shape:
expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item()
if abs(received_sum - expected_sum) < 0.01:
@@ -478,11 +563,9 @@ def test_nccl_weight_transfer_between_processes():
master_port = get_open_port()
world_size = 2 # 1 trainer + 1 inference worker
# Tensor to transfer: 100x100 ones
tensor_shape = [100, 100]
tensor_dtype = "float32"
# Start both tasks concurrently - Ray assigns GPUs automatically
inference_future = inference_receive_tensor.remote(
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
@@ -490,7 +573,6 @@ def test_nccl_weight_transfer_between_processes():
master_address, master_port, world_size, tensor_shape, tensor_dtype
)
# Wait for both to complete
trainer_result, result = ray.get([trainer_future, inference_future])
assert trainer_result, "Trainer should complete successfully"
@@ -514,10 +596,12 @@ def trainer_broadcast_sparse_tensor(
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
)
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseWeightPatch,
)
pg = StatelessProcessGroup.create(
@@ -533,7 +617,7 @@ def trainer_broadcast_sparse_tensor(
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device=device),
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device=device),
)
NCCLWeightTransferEngine.trainer_send_sparse_weights(
SparseNCCLWeightTransferEngine.trainer_send_weights(
iter([patch]),
NCCLTrainerSendWeightsArgs(group=comm),
)
@@ -547,7 +631,7 @@ def inference_receive_sparse_tensor(
master_port: int,
world_size: int,
) -> dict:
"""Inference task that receives sparse patches via NCCLWeightTransferEngine."""
"""Inference task that receives sparse patches via the sparse engine."""
from unittest.mock import MagicMock
import torch
@@ -556,22 +640,40 @@ def inference_receive_sparse_tensor(
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
from vllm.distributed.weight_transfer.sparse_nccl_engine import (
SparseNCCLWeightTransferEngine,
SparseNCCLWeightTransferUpdateInfo,
)
config = WeightTransferConfig(backend="nccl")
config = WeightTransferConfig(backend="sparse_nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = NCCLWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
# Real module holding the target parameter the patch will modify.
model = torch.nn.Module()
model.register_parameter(
"w", torch.nn.Parameter(torch.zeros(30, device="cuda"), requires_grad=False)
)
model.get_parameter = lambda name: model.w
update_info = SparseNCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
)
engine = SparseNCCLWeightTransferEngine(config, vllm_config, "cuda", model)
from vllm.distributed.weight_transfer.nccl_common import (
NCCLWeightTransferInitInfo,
)
engine.init_transfer_engine(
NCCLWeightTransferInitInfo(
master_address=master_address,
@@ -580,32 +682,18 @@ def inference_receive_sparse_tensor(
world_size=world_size,
)
)
target = torch.zeros(30, dtype=torch.float32, device=device)
def apply_sparse_patches(patches: list[SparseWeightPatch]):
for patch in patches:
target.index_copy_(0, patch.indices.to(torch.long), patch.values)
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
update_kind="sparse_flat",
)
engine.receive_sparse_weights(update_info, apply_sparse_patches)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
expected = torch.zeros(30, dtype=torch.float32, device=device)
expected[[1, 7, 25]] = torch.tensor(
[10.0, 20.0, 30.0], dtype=torch.float32, device=device
)
success = torch.equal(target, expected)
success = torch.equal(model.w.data, expected)
engine.shutdown()
return {
"success": success,
"selected_values": target[[1, 7, 25]].cpu().tolist(),
"selected_values": model.w.data[[1, 7, 25]].cpu().tolist(),
}
@@ -644,11 +732,9 @@ class TestIPCWeightTransferUpdateInfoValidation:
"""Test IPCWeightTransferUpdateInfo dataclass validation."""
def test_valid_update_info(self):
"""Test creating valid IPCWeightTransferUpdateInfo."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
# Create a dummy tensor and IPC handle
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
@@ -666,7 +752,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
assert len(info.ipc_handles) == 1
def test_mismatched_dtype_names_raises(self):
"""Test that mismatched dtype_names length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -684,7 +769,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_mismatched_shapes_raises(self):
"""Test that mismatched shapes length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -702,7 +786,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_mismatched_ipc_handles_raises(self):
"""Test that mismatched ipc_handles length raises ValueError."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -719,44 +802,7 @@ class TestIPCWeightTransferUpdateInfoValidation:
ipc_handles=ipc_handles,
)
def test_sparse_update_kind_rejected(self):
"""Test that IPC backend rejects sparse update metadata."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
dummy_tensor = torch.ones(10, 10, device="cuda:0")
ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
ipc_handles = [{gpu_uuid: ipc_handle}]
with pytest.raises(NotImplementedError, match="dense updates"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[1],
ipc_handles=ipc_handles,
update_kind="sparse_flat",
)
def test_sparse_methods_not_supported(self):
"""Test that IPC engine inherits sparse rejection from the base class."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.receive_sparse_weights(MagicMock(), lambda _: None)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.trainer_send_sparse_weights(
iter([]),
{"mode": "http", "url": "http://localhost:8000"},
)
def test_valid_update_info_from_pickled(self, monkeypatch):
"""Test creating IPCWeightTransferUpdateInfo from pickled handles."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -779,7 +825,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
assert info.ipc_handles_pickled is None
def test_pickled_requires_insecure_serialization_flag(self, monkeypatch):
"""Test that pickled handles are rejected unless env flag is enabled."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")
with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"):
@@ -791,7 +836,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
@@ -812,7 +856,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_neither_handles_nor_pickled_raises(self):
"""Test that providing neither ipc_handles nor ipc_handles_pickled raises."""
with pytest.raises(ValueError, match="must be provided"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
@@ -821,7 +864,6 @@ class TestIPCWeightTransferUpdateInfoValidation:
)
def test_empty_lists_valid(self):
"""Test that empty lists are valid."""
info = IPCWeightTransferUpdateInfo(
names=[],
dtype_names=[],
@@ -837,18 +879,21 @@ class TestIPCWeightTransferUpdateInfoValidation:
class TestIPCEngineParsing:
"""Test IPCWeightTransferEngine parsing methods."""
def _make_engine(self):
config = WeightTransferConfig(backend="ipc")
return IPCWeightTransferEngine(
config,
create_mock_vllm_config(),
"cuda",
MagicMock(spec=torch.nn.Module),
)
def test_parse_update_info_valid(self):
"""Test parsing valid update info dict."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
# Create dummy IPC handles
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
dummy_tensor2 = torch.ones(50, device="cuda:0")
_, ipc_args1 = reduce_tensor(dummy_tensor1)
@@ -872,17 +917,12 @@ class TestIPCEngineParsing:
assert len(update_info.ipc_handles) == 2
def test_parse_update_info_pickled(self, monkeypatch):
"""Test parsing update info with pickled IPC handles (HTTP path)."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
dummy_tensor1 = torch.ones(100, 100, device="cuda:0")
dummy_tensor2 = torch.ones(50, device="cuda:0")
@@ -909,12 +949,7 @@ class TestIPCEngineParsing:
assert gpu_uuid in update_info.ipc_handles[1]
def test_parse_update_info_ignores_none_pickled_handles(self):
"""Test Ray/asdict payloads with a null pickled field use ipc_handles."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
ipc_handles = [{"gpu-uuid": ("ipc-args",)}]
update_info = engine.parse_update_info(
@@ -931,15 +966,10 @@ class TestIPCEngineParsing:
assert update_info.ipc_handles == ipc_handles
def test_parse_update_info_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
engine = self._make_engine()
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
@@ -981,9 +1011,6 @@ class TrainerActor:
self.tensor = torch.ones(tensor_shape, dtype=dtype, device=device)
self.tensor.fill_(42.0) # Fill with 42 to verify correct transfer
# Create IPC handle (tensor must stay alive for IPC to work)
# reduce_tensor returns (rebuild_func, args); we only send args
# since the receiver imports rebuild_cuda_tensor directly.
_, ipc_args = reduce_tensor(self.tensor)
gpu_uuid = get_physical_gpu_id(device.index)
@@ -1007,6 +1034,7 @@ def inference_receive_ipc_tensor(
mode: str = "ray",
) -> dict:
"""Inference task that receives tensor via IPCWeightTransferEngine."""
import contextlib
import os
# Worker-side: ipc_handles_pickled is deserialized via pickle.
@@ -1025,30 +1053,36 @@ def inference_receive_ipc_tensor(
IPCWeightTransferEngine,
)
# Create engine with mock parallel config
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="ipc")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
recorder = Recorder()
engine = IPCWeightTransferEngine(config, vllm_config, "cuda", recorder)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
# Initialize the engine (no-op for IPC)
init_info = IPCWeightTransferInitInfo()
engine.init_transfer_engine(init_info)
# Receive weights with a no-op load_weights that captures the tensor
received_tensors = []
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
for name, tensor in weights:
# Clone tensor to keep it after engine cleans up
received_tensors.append((name, tensor.clone()))
ipc_handles = [{ipc_handle_dict["gpu_uuid"]: ipc_handle_dict["ipc_handle"]}]
if mode == "ray":
@@ -1059,7 +1093,6 @@ def inference_receive_ipc_tensor(
"ipc_handles": ipc_handles,
}
elif mode == "http":
# Simulate HTTP transport: pickle + base64 encode handles
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
update_dict = {
"names": ["test.weight"],
@@ -1071,19 +1104,17 @@ def inference_receive_ipc_tensor(
raise ValueError(f"Unknown mode: {mode}")
update_info = engine.parse_update_info(update_dict)
engine.receive_weights(update_info, noop_load_weights)
engine.receive_weights(update_info)
torch.accelerator.synchronize()
# Verify we received the tensor
success = False
received_shape = None
received_sum = None
if len(received_tensors) == 1:
name, tensor = received_tensors[0]
if len(recorder.received) == 1:
name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
# Check shape matches and values are all 42s (trainer sends 42s)
if received_shape == ipc_handle_dict["shape"]:
expected_sum = 42.0 * torch.tensor(ipc_handle_dict["shape"]).prod().item()
if abs(received_sum - expected_sum) < 0.01:
@@ -1104,23 +1135,12 @@ def inference_receive_ipc_tensor(
)
@pytest.mark.parametrize("mode", ["ray", "http"])
def test_ipc_weight_transfer_between_processes(mode: str):
"""Test IPC weight transfer from trainer to inference process using Ray.
Parametrized over transport modes:
- 'ray': ipc_handles passed directly.
- 'http': ipc_handles pickled + base64-encoded, deserialized in
parse_update_info before constructing the dataclass.
IPC requires same-GPU access, so we use a placement group to co-locate
the trainer actor and inference task on the same GPU.
"""
"""Test IPC weight transfer from trainer to inference process using Ray."""
from ray.util.placement_group import placement_group
from ray.util.scheduling_strategies import PlacementGroupSchedulingStrategy
_init_ray_for_weight_transfer()
# Create a placement group to ensure both processes are on the same GPU
# Use fractional GPUs so both tasks can share the same GPU bundle
pg = placement_group([{"GPU": 1, "CPU": 2}])
ray.get(pg.ready())
@@ -1129,20 +1149,15 @@ def test_ipc_weight_transfer_between_processes(mode: str):
placement_group_capture_child_tasks=True,
)
# Tensor to transfer: 100x100 filled with 42s
tensor_shape = [100, 100]
tensor_dtype = "float32"
# Create trainer actor that holds the tensor and IPC handle (stays alive)
trainer_actor = TrainerActor.options( # type: ignore[attr-defined]
scheduling_strategy=scheduling_strategy
).remote(tensor_shape, tensor_dtype)
# Get IPC handle dict (tensor stays alive in trainer actor)
ipc_handle_dict = ray.get(trainer_actor.get_ipc_handle_dict.remote())
# Receive tensor in inference process using IPC handles (on same GPU)
# Trainer actor stays alive during this operation
inference_result = ray.get(
inference_receive_ipc_tensor.options(
scheduling_strategy=scheduling_strategy
@@ -1162,12 +1177,10 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
config, create_mock_vllm_config(), "cuda", MagicMock(spec=torch.nn.Module)
)
# Create IPC handle with wrong GPU UUID
dummy_tensor = torch.ones(10, 10, device="cuda:0")
_, ipc_handle = reduce_tensor(dummy_tensor)
wrong_uuid = "wrong-uuid-12345"
@@ -1181,4 +1194,4 @@ def test_ipc_receive_weights_missing_gpu_uuid_raises():
)
with pytest.raises(ValueError, match="IPC handle not found"):
engine.receive_weights(update_info, lambda x: None)
engine.receive_weights(update_info)
@@ -8,7 +8,6 @@ actual NCCL communication.
"""
import os
from collections.abc import Callable
from dataclasses import dataclass
from unittest.mock import patch
@@ -48,7 +47,6 @@ class MockUpdateInfo(WeightTransferUpdateInfo):
names: list[str] | None = None
dtype_names: list[str] | None = None
shapes: list[list[int]] | None = None
num_updates_list: list[int] | None = None
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
@@ -59,16 +57,20 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
# Class-level tracking for verification across processes
init_transfer_engine_called: bool = False
start_called: bool = False
receive_weights_called: bool = False
finish_called: bool = False
shutdown_called: bool = False
last_init_info: MockInitInfo | None = None
last_update_info: MockUpdateInfo | None = None
def __init__(self, config, parallel_config, model):
super().__init__(config, parallel_config, model)
def __init__(self, config, vllm_config, device, model):
super().__init__(config, vllm_config, device, model)
# Reset tracking on init
MockWeightTransferEngine.init_transfer_engine_called = False
MockWeightTransferEngine.start_called = False
MockWeightTransferEngine.receive_weights_called = False
MockWeightTransferEngine.finish_called = False
MockWeightTransferEngine.shutdown_called = False
MockWeightTransferEngine.last_init_info = None
MockWeightTransferEngine.last_update_info = None
@@ -77,37 +79,28 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
MockWeightTransferEngine.init_transfer_engine_called = True
MockWeightTransferEngine.last_init_info = init_info
def receive_weights(
self,
update_info: MockUpdateInfo,
load_weights: Callable[[list[tuple[str, torch.Tensor]]], None],
) -> None:
MockWeightTransferEngine.receive_weights_called = True
MockWeightTransferEngine.last_update_info = update_info
# Simulate loading weights by calling load_weights with empty list
# (In real implementation, this would receive and load actual weights)
load_weights([])
def start_weight_update(self) -> None:
MockWeightTransferEngine.start_called = True
def receive_sparse_weights(
self,
update_info: MockUpdateInfo,
apply_patches: Callable[[list], None],
) -> None:
def finish_weight_update(self) -> None:
MockWeightTransferEngine.finish_called = True
def receive_weights(self, update_info: MockUpdateInfo) -> None:
MockWeightTransferEngine.receive_weights_called = True
MockWeightTransferEngine.last_update_info = update_info
apply_patches([])
def shutdown(self) -> None:
MockWeightTransferEngine.shutdown_called = True
def trainer_send_weights(self, *args, **kwargs):
@staticmethod
def trainer_send_weights(*args, **kwargs):
"""Mock method to simulate trainer sending weights."""
pass
def mock_create_engine(config, parallel_config, model):
def mock_create_engine(config, vllm_config, device, model):
"""Mock factory function that returns our mock engine."""
return MockWeightTransferEngine(config, parallel_config, model)
return MockWeightTransferEngine(config, vllm_config, device, model)
# --- Tests ---
@@ -208,7 +201,7 @@ def test_update_weights_calls_engine():
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
llm.start_weight_update(is_checkpoint_format=True)
llm.start_weight_update()
# Call update_weights
test_names = ["layer.weight", "layer.bias"]
@@ -243,61 +236,6 @@ def test_update_weights_calls_engine():
llm.finish_weight_update()
@create_new_process_for_each_test()
def test_update_weights_passes_sparse_metadata():
"""Test sparse update metadata is forwarded unchanged to the engine."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
):
llm = LLM(
model=MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
llm.start_weight_update(is_checkpoint_format=False)
llm.update_weights(
WeightTransferUpdateRequest(
update_info={
"names": ["layer.weight"],
"dtype_names": ["bfloat16"],
"shapes": [[100]],
"num_updates_list": [3],
"update_kind": "sparse_flat",
}
)
)
def check_sparse_update_called(self):
engine = self.weight_transfer_engine
if not engine.receive_weights_called:
return None
info = engine.last_update_info
return (
info.update_kind,
info.num_updates_list,
)
results = llm.collective_rpc(check_sparse_update_called)
for result in results:
assert result == ("sparse_flat", [3])
llm.finish_weight_update()
@create_new_process_for_each_test()
def test_full_weight_transfer_flow():
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
@@ -327,7 +265,7 @@ def test_full_weight_transfer_flow():
)
# Step 2: Start weight update
llm.start_weight_update(is_checkpoint_format=True)
llm.start_weight_update()
# Step 3: Update weights
llm.update_weights(
-69
View File
@@ -7,7 +7,6 @@ from unittest.mock import Mock
import numpy as np
import pytest
import torch
import torch.nn as nn
import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module
from vllm.config import (
@@ -23,7 +22,6 @@ from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.lora.layers import LoRAMappingType
from vllm.lora.request import LoRARequest
from vllm.model_executor.layers.attention import Attention
@@ -864,73 +862,6 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
assert torch.equal(passed_draft_probs, expected_draft_probs)
def test_apply_sparse_weight_patches_updates_only_selected_entries():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(6, dtype=torch.float32))
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1, 4], dtype=torch.int32),
values=torch.tensor([3.5, -2.0], dtype=torch.float32),
)
]
)
expected = torch.tensor([0.0, 3.5, 0.0, 0.0, -2.0, 0.0], dtype=torch.float32)
assert torch.equal(runner.get_model().weight.data, expected)
def test_apply_sparse_weight_patches_rejects_mismatched_lengths():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
with pytest.raises(ValueError, match="matching lengths"):
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1, 2], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
]
)
def test_apply_sparse_weight_patches_rejects_non_contiguous_param():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t()
)
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
with pytest.raises(NotImplementedError, match="contiguous params"):
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
]
)
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
torch.set_default_dtype(torch.float16)
layer_0 = "model.layers.0.self_attn.attn"
@@ -1,155 +1,94 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for GPUWorker weight-transfer pass-through behavior.
from types import SimpleNamespace
from unittest.mock import MagicMock
The worker no longer contains transport, layerwise, or sparse logic: it only
delegates to the configured weight transfer engine and tracks whether an update
session is active. These tests verify that delegation and the session guard.
"""
import pytest
import torch
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferEngine
from vllm.v1.worker.gpu_worker import Worker
def _make_nccl_engine() -> NCCLWeightTransferEngine:
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
return NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
parallel_config,
MagicMock(spec=torch.nn.Module),
)
class _RecordingEngine:
"""Minimal stand-in for a weight transfer engine."""
def __init__(self, raise_on_update: bool = False):
self.raise_on_update = raise_on_update
self.started = False
self.finished = False
self.update_calls: list[dict] = []
def start_weight_update(self) -> None:
self.started = True
def update_weights(self, update_info: dict) -> None:
self.update_calls.append(update_info)
if self.raise_on_update:
raise ValueError("boom")
def finish_weight_update(self) -> None:
self.finished = True
def test_update_weights_sparse_dispatches_to_sparse_receive(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
def _make_worker(engine: _RecordingEngine | None) -> Worker:
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
applied_patches = []
def apply_sparse_weight_patches(patches):
applied_patches.extend(patches)
worker.model_runner = SimpleNamespace(
apply_sparse_weight_patches=apply_sparse_weight_patches,
)
received_kinds = []
def receive_sparse_weights(update_info, apply_patches):
received_kinds.append(update_info.update_kind)
apply_patches(
[
SparseWeightPatch(
name="layer.weight",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([2.0], dtype=torch.float32),
)
]
)
worker.weight_transfer_engine.receive_sparse_weights = receive_sparse_weights
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
assert received_kinds == ["sparse_flat"]
assert len(applied_patches) == 1
assert torch.equal(applied_patches[0].indices, torch.tensor([1], dtype=torch.int32))
worker.weight_transfer_engine = engine
worker._weight_update_active = False
return worker
def test_update_weights_sparse_rejects_tp_or_pp(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
def test_start_update_finish_delegates_to_engine():
engine = _RecordingEngine()
worker = _make_worker(engine)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=2)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
worker.model_runner = SimpleNamespace(apply_sparse_weight_patches=lambda _: None)
Worker.start_weight_update(worker)
assert engine.started is True
assert worker._weight_update_active is True
with pytest.raises(NotImplementedError, match="TP=1 and PP=1"):
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
Worker.update_weights(worker, {"names": ["w"]})
assert engine.update_calls == [{"names": ["w"]}]
assert worker._weight_update_active is True
Worker.finish_weight_update(worker)
assert engine.finished is True
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_update_weights_sparse_rejects_checkpoint_format(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
def test_double_start_raises():
worker = _make_worker(_RecordingEngine())
Worker.start_weight_update(worker)
with pytest.raises(RuntimeError, match="already"):
Worker.start_weight_update(worker)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = True
worker.model_runner = SimpleNamespace(model=MagicMock())
with pytest.raises(ValueError, match="start_weight_update"):
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
def test_update_without_start_raises():
worker = _make_worker(_RecordingEngine())
with pytest.raises(RuntimeError, match="start_weight_update must be called"):
Worker.update_weights(worker, {"names": ["w"]})
def test_finish_without_start_raises():
worker = _make_worker(_RecordingEngine())
with pytest.raises(RuntimeError, match="without a matching"):
Worker.finish_weight_update(worker)
def test_update_resets_active_on_error():
engine = _RecordingEngine(raise_on_update=True)
worker = _make_worker(engine)
Worker.start_weight_update(worker)
with pytest.raises(ValueError, match="boom"):
Worker.update_weights(worker, {"names": ["w"]})
# A failed update ends the session so the next start is clean.
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_update_weights_resets_state_when_update_info_is_invalid(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
with pytest.raises(ValueError, match="cannot be empty"):
Worker.update_weights(
worker,
{
"names": [],
"dtype_names": [],
"shapes": [],
"num_updates_list": [],
"update_kind": "sparse_flat",
},
)
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_missing_engine_raises():
worker = _make_worker(None)
with pytest.raises(RuntimeError, match="Weight transfer not configured"):
Worker.start_weight_update(worker)