[Frontend][Core] Add sparse NCCL weight transfer support for in-place updates (#40096)

Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com>
Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
Siddharth Bedekar
2026-06-01 15:37:30 -04:00
committed by GitHub
co-authored by OpenAI Codex
parent 182c67daf1
commit 266b9d9c64
12 changed files with 1423 additions and 75 deletions
+330 -3
View File
@@ -18,6 +18,7 @@ from torch.multiprocessing.reductions import reduce_tensor
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.ipc_engine import (
IPCWeightTransferEngine,
IPCWeightTransferInitInfo,
@@ -89,6 +90,67 @@ class TestNCCLWeightTransferUpdateInfoValidation:
)
assert len(info.names) == 0
def test_valid_sparse_update_info(self):
"""Test creating valid sparse NCCL update info."""
info = NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "bfloat16"],
shapes=[[10, 10], [10]],
num_updates_list=[4, 2],
update_kind="sparse_flat",
)
assert info.update_kind == "sparse_flat"
assert info.num_updates_list == [4, 2]
def test_sparse_update_requires_num_updates_list(self):
with pytest.raises(ValueError, match="`num_updates_list` is required"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
update_kind="sparse_flat",
)
def test_sparse_update_rejects_empty_num_updates_list(self):
with pytest.raises(ValueError, match="cannot be empty"):
NCCLWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
num_updates_list=[],
update_kind="sparse_flat",
)
def test_sparse_update_rejects_packed(self):
with pytest.raises(ValueError, match="cannot be combined with `packed=True`"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
update_kind="sparse_flat",
packed=True,
)
def test_sparse_update_rejects_mismatched_num_updates(self):
with pytest.raises(ValueError, match="`num_updates_list`"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight", "layer.bias"],
dtype_names=["float32", "float32"],
shapes=[[10, 10], [10]],
num_updates_list=[3],
update_kind="sparse_flat",
)
def test_dense_update_rejects_sparse_metadata(self):
with pytest.raises(ValueError, match="Sparse metadata"):
NCCLWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[3],
)
# --- Unit Tests: Engine Parsing ---
@@ -222,6 +284,27 @@ def test_nccl_receive_weights_without_init_raises():
engine.receive_weights(update_info, lambda x: None)
def test_nccl_receive_sparse_weights_without_init_raises():
"""Test that sparse receive raises if init_transfer_engine wasn't called."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
config = WeightTransferConfig(backend="nccl")
parallel_config = create_mock_parallel_config()
engine = NCCLWeightTransferEngine(config, parallel_config)
update_info = NCCLWeightTransferUpdateInfo(
names=["w"],
dtype_names=["float32"],
shapes=[[10]],
num_updates_list=[2],
update_kind="sparse_flat",
)
with pytest.raises(RuntimeError, match="not initialized"):
engine.receive_sparse_weights(update_info, lambda x: None)
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
@@ -379,6 +462,136 @@ def test_nccl_weight_transfer_between_processes():
)
@ray.remote(num_gpus=1)
def trainer_broadcast_sparse_tensor(
master_address: str,
master_port: int,
world_size: int,
) -> bool:
"""Trainer task that broadcasts sparse patches via NCCL."""
import torch
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
from vllm.distributed.utils import StatelessProcessGroup
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerSendWeightsArgs,
NCCLWeightTransferEngine,
)
pg = StatelessProcessGroup.create(
host=master_address,
port=master_port,
rank=0,
world_size=world_size,
)
comm = PyNcclCommunicator(pg, device=0)
patch = SparseWeightPatch(
name="test.weight",
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device="cuda:0"),
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device="cuda:0"),
)
NCCLWeightTransferEngine.trainer_send_sparse_weights(
iter([patch]),
NCCLTrainerSendWeightsArgs(group=comm),
)
torch.accelerator.synchronize()
return True
@ray.remote(num_gpus=1)
def inference_receive_sparse_tensor(
master_address: str,
master_port: int,
world_size: int,
) -> dict:
"""Inference task that receives sparse patches via NCCLWeightTransferEngine."""
from unittest.mock import MagicMock
import torch
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
config = WeightTransferConfig(backend="nccl")
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
engine = NCCLWeightTransferEngine(config, parallel_config)
engine.init_transfer_engine(
NCCLWeightTransferInitInfo(
master_address=master_address,
master_port=master_port,
rank_offset=1,
world_size=world_size,
)
)
target = torch.zeros(30, dtype=torch.float32, device="cuda")
def apply_sparse_patches(patches: list[SparseWeightPatch]):
for patch in patches:
target.index_copy_(0, patch.indices.to(torch.long), patch.values)
update_info = NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=["float32"],
shapes=[[30]],
num_updates_list=[3],
update_kind="sparse_flat",
)
engine.receive_sparse_weights(update_info, apply_sparse_patches)
torch.accelerator.synchronize()
expected = torch.zeros(30, dtype=torch.float32, device="cuda")
expected[[1, 7, 25]] = torch.tensor(
[10.0, 20.0, 30.0], dtype=torch.float32, device="cuda"
)
success = torch.equal(target, expected)
engine.shutdown()
return {
"success": success,
"selected_values": target[[1, 7, 25]].cpu().tolist(),
}
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL sparse weight transfer test.",
)
def test_nccl_sparse_weight_transfer_between_processes():
"""Test NCCL sparse weight transfer from trainer to inference process."""
ray.init(ignore_reinit_error=True)
master_address = "127.0.0.1"
master_port = get_open_port()
world_size = 2
inference_future = inference_receive_sparse_tensor.remote(
master_address, master_port, world_size
)
trainer_future = trainer_broadcast_sparse_tensor.remote(
master_address, master_port, world_size
)
trainer_result, result = ray.get([trainer_future, inference_future])
assert trainer_result, "Trainer should complete successfully"
assert result["success"], (
"Sparse weight transfer failed. "
f"Received selected values: {result['selected_values']}"
)
# --- Unit Tests: IPCWeightTransferUpdateInfo Validation ---
@@ -461,9 +674,101 @@ class TestIPCWeightTransferUpdateInfoValidation:
ipc_handles=ipc_handles,
)
def test_missing_ipc_handles_raises(self):
"""Test that omitting ipc_handles raises TypeError."""
with pytest.raises(TypeError):
def test_sparse_update_kind_rejected(self):
"""Test that IPC backend rejects sparse update metadata."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
dummy_tensor = torch.ones(10, 10, device="cuda:0")
ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
ipc_handles = [{gpu_uuid: ipc_handle}]
with pytest.raises(NotImplementedError, match="dense updates"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
num_updates_list=[1],
ipc_handles=ipc_handles,
update_kind="sparse_flat",
)
def test_sparse_methods_not_supported(self):
"""Test that IPC engine inherits sparse rejection from the base class."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.receive_sparse_weights(MagicMock(), lambda _: None)
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
engine.trainer_send_sparse_weights(
iter([]),
{"mode": "http", "url": "http://localhost:8000"},
)
def test_valid_update_info_from_pickled(self, monkeypatch):
"""Test creating IPCWeightTransferUpdateInfo from pickled handles."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
dummy_tensor = torch.ones(10, 10, device="cuda:0")
ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
ipc_handles = [{gpu_uuid: ipc_handle}]
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
info = IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
ipc_handles_pickled=pickled,
)
assert info.ipc_handles == ipc_handles
assert info.ipc_handles_pickled is None
def test_pickled_requires_insecure_serialization_flag(self, monkeypatch):
"""Test that pickled handles are rejected unless env flag is enabled."""
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")
with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"):
IPCWeightTransferUpdateInfo(
names=[],
dtype_names=[],
shapes=[],
ipc_handles_pickled=base64.b64encode(pickle.dumps([])).decode("utf-8"),
)
def test_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
dummy_tensor = torch.ones(10, 10, device="cuda:0")
ipc_handle = reduce_tensor(dummy_tensor)
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
ipc_handles = [{gpu_uuid: ipc_handle}]
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
with pytest.raises(ValueError, match="Cannot specify both"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
shapes=[[10, 10]],
ipc_handles=ipc_handles,
ipc_handles_pickled=pickled,
)
def test_neither_handles_nor_pickled_raises(self):
"""Test that providing neither ipc_handles nor ipc_handles_pickled raises."""
with pytest.raises(ValueError, match="must be provided"):
IPCWeightTransferUpdateInfo(
names=["layer.weight"],
dtype_names=["float32"],
@@ -558,6 +863,28 @@ class TestIPCEngineParsing:
assert gpu_uuid in update_info.ipc_handles[0]
assert gpu_uuid in update_info.ipc_handles[1]
def test_parse_update_info_ignores_none_pickled_handles(self):
"""Test Ray/asdict payloads with a null pickled field use ipc_handles."""
config = WeightTransferConfig(backend="ipc")
parallel_config = create_mock_parallel_config()
engine = IPCWeightTransferEngine(
config, parallel_config, MagicMock(spec=torch.nn.Module)
)
ipc_handles = [{"gpu-uuid": ("ipc-args",)}]
update_info = engine.parse_update_info(
{
"names": ["w1"],
"dtype_names": ["float32"],
"shapes": [[1]],
"ipc_handles": ipc_handles,
"ipc_handles_pickled": None,
}
)
assert isinstance(update_info, IPCWeightTransferUpdateInfo)
assert update_info.ipc_handles == ipc_handles
def test_parse_update_info_both_handles_and_pickled_raises(self):
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
if torch.accelerator.device_count() < 1:
@@ -48,6 +48,7 @@ class MockUpdateInfo(WeightTransferUpdateInfo):
names: list[str] | None = None
dtype_names: list[str] | None = None
shapes: list[list[int]] | None = None
num_updates_list: list[int] | None = None
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
@@ -87,6 +88,15 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
# (In real implementation, this would receive and load actual weights)
load_weights([])
def receive_sparse_weights(
self,
update_info: MockUpdateInfo,
apply_patches: Callable[[list], None],
) -> None:
MockWeightTransferEngine.receive_weights_called = True
MockWeightTransferEngine.last_update_info = update_info
apply_patches([])
def shutdown(self) -> None:
MockWeightTransferEngine.shutdown_called = True
@@ -198,8 +208,6 @@ def test_update_weights_calls_engine():
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
# Start weight update (required before update_weights)
llm.start_weight_update(is_checkpoint_format=True)
# Call update_weights
@@ -232,14 +240,67 @@ def test_update_weights_calls_engine():
assert dtypes == test_dtypes
assert shapes == test_shapes
# Finish weight update
llm.finish_weight_update()
@create_new_process_for_each_test()
def test_update_weights_passes_sparse_metadata():
"""Test sparse update metadata is forwarded unchanged to the engine."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
with patch(
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
mock_create_engine,
):
llm = LLM(
model=MODEL_NAME,
enforce_eager=True,
load_format="dummy",
tensor_parallel_size=1,
weight_transfer_config=WeightTransferConfig(backend="nccl"),
)
llm.init_weight_transfer_engine(
WeightTransferInitRequest(init_info={"test_param": "init"})
)
llm.start_weight_update(is_checkpoint_format=False)
llm.update_weights(
WeightTransferUpdateRequest(
update_info={
"names": ["layer.weight"],
"dtype_names": ["bfloat16"],
"shapes": [[100]],
"num_updates_list": [3],
"update_kind": "sparse_flat",
}
)
)
def check_sparse_update_called(self):
engine = self.weight_transfer_engine
if not engine.receive_weights_called:
return None
info = engine.last_update_info
return (
info.update_kind,
info.num_updates_list,
)
results = llm.collective_rpc(check_sparse_update_called)
for result in results:
assert result == ("sparse_flat", [3])
llm.finish_weight_update()
@create_new_process_for_each_test()
def test_full_weight_transfer_flow():
"""Test the complete weight transfer flow:
init -> start -> update -> finish."""
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
if torch.accelerator.device_count() < 1:
pytest.skip("Need at least 1 GPU for this test")
+69
View File
@@ -7,6 +7,7 @@ from unittest.mock import Mock
import numpy as np
import pytest
import torch
import torch.nn as nn
import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module
from vllm.config import (
@@ -22,6 +23,7 @@ from vllm.distributed.parallel_state import (
init_distributed_environment,
initialize_model_parallel,
)
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.model_executor.layers.attention import Attention
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
from vllm.platforms import current_platform
@@ -784,6 +786,73 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
assert torch.equal(passed_draft_probs, expected_draft_probs)
def test_apply_sparse_weight_patches_updates_only_selected_entries():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(6, dtype=torch.float32))
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1, 4], dtype=torch.int32),
values=torch.tensor([3.5, -2.0], dtype=torch.float32),
)
]
)
expected = torch.tensor([0.0, 3.5, 0.0, 0.0, -2.0, 0.0], dtype=torch.float32)
assert torch.equal(runner.get_model().weight.data, expected)
def test_apply_sparse_weight_patches_rejects_mismatched_lengths():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
with pytest.raises(ValueError, match="matching lengths"):
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1, 2], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
]
)
def test_apply_sparse_weight_patches_rejects_non_contiguous_param():
class DummyModel(nn.Module):
def __init__(self):
super().__init__()
self.weight = nn.Parameter(
torch.arange(12, dtype=torch.float32).view(3, 4).t()
)
runner = object.__new__(GPUModelRunner)
runner.model = DummyModel()
with pytest.raises(NotImplementedError, match="contiguous params"):
runner.apply_sparse_weight_patches(
[
SparseWeightPatch(
name="weight",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([1.0], dtype=torch.float32),
)
]
)
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
torch.set_default_dtype(torch.float16)
layer_0 = "model.layers.0.self_attn.attn"
@@ -0,0 +1,155 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from unittest.mock import MagicMock
import pytest
import torch
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.base import SparseWeightPatch
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferEngine
from vllm.v1.worker.gpu_worker import Worker
def _make_nccl_engine() -> NCCLWeightTransferEngine:
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
return NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
parallel_config,
MagicMock(spec=torch.nn.Module),
)
def test_update_weights_sparse_dispatches_to_sparse_receive(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
applied_patches = []
def apply_sparse_weight_patches(patches):
applied_patches.extend(patches)
worker.model_runner = SimpleNamespace(
apply_sparse_weight_patches=apply_sparse_weight_patches,
)
received_kinds = []
def receive_sparse_weights(update_info, apply_patches):
received_kinds.append(update_info.update_kind)
apply_patches(
[
SparseWeightPatch(
name="layer.weight",
indices=torch.tensor([1], dtype=torch.int32),
values=torch.tensor([2.0], dtype=torch.float32),
)
]
)
worker.weight_transfer_engine.receive_sparse_weights = receive_sparse_weights
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
assert received_kinds == ["sparse_flat"]
assert len(applied_patches) == 1
assert torch.equal(applied_patches[0].indices, torch.tensor([1], dtype=torch.int32))
def test_update_weights_sparse_rejects_tp_or_pp(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=2)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
worker.model_runner = SimpleNamespace(apply_sparse_weight_patches=lambda _: None)
with pytest.raises(NotImplementedError, match="TP=1 and PP=1"):
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_update_weights_sparse_rejects_checkpoint_format(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = True
worker.model_runner = SimpleNamespace(model=MagicMock())
with pytest.raises(ValueError, match="start_weight_update"):
Worker.update_weights(
worker,
{
"names": ["layer.weight"],
"dtype_names": ["float32"],
"shapes": [[4]],
"num_updates_list": [1],
"update_kind": "sparse_flat",
},
)
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True
def test_update_weights_resets_state_when_update_info_is_invalid(monkeypatch):
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
worker = object.__new__(Worker)
worker.device = "cpu"
worker.parallel_config = SimpleNamespace(world_size=1)
worker.weight_transfer_engine = _make_nccl_engine()
worker._weight_update_active = True
worker._is_checkpoint_format = False
with pytest.raises(ValueError, match="cannot be empty"):
Worker.update_weights(
worker,
{
"names": [],
"dtype_names": [],
"shapes": [],
"num_updates_list": [],
"update_kind": "sparse_flat",
},
)
assert worker._weight_update_active is False
assert worker._is_checkpoint_format is True