[Frontend][Core] Add sparse NCCL weight transfer support for in-place updates (#40096)
Signed-off-by: Siddharth Bedekar <bedeksid@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
182c67daf1
commit
266b9d9c64
@@ -18,6 +18,7 @@ from torch.multiprocessing.reductions import reduce_tensor
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.ipc_engine import (
|
||||
IPCWeightTransferEngine,
|
||||
IPCWeightTransferInitInfo,
|
||||
@@ -89,6 +90,67 @@ class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
)
|
||||
assert len(info.names) == 0
|
||||
|
||||
def test_valid_sparse_update_info(self):
|
||||
"""Test creating valid sparse NCCL update info."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "bfloat16"],
|
||||
shapes=[[10, 10], [10]],
|
||||
num_updates_list=[4, 2],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
assert info.update_kind == "sparse_flat"
|
||||
assert info.num_updates_list == [4, 2]
|
||||
|
||||
def test_sparse_update_requires_num_updates_list(self):
|
||||
with pytest.raises(ValueError, match="`num_updates_list` is required"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_sparse_update_rejects_empty_num_updates_list(self):
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=[],
|
||||
dtype_names=[],
|
||||
shapes=[],
|
||||
num_updates_list=[],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_sparse_update_rejects_packed(self):
|
||||
with pytest.raises(ValueError, match="cannot be combined with `packed=True`"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
num_updates_list=[3],
|
||||
update_kind="sparse_flat",
|
||||
packed=True,
|
||||
)
|
||||
|
||||
def test_sparse_update_rejects_mismatched_num_updates(self):
|
||||
with pytest.raises(ValueError, match="`num_updates_list`"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "float32"],
|
||||
shapes=[[10, 10], [10]],
|
||||
num_updates_list=[3],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_dense_update_rejects_sparse_metadata(self):
|
||||
with pytest.raises(ValueError, match="Sparse metadata"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
num_updates_list=[3],
|
||||
)
|
||||
|
||||
|
||||
# --- Unit Tests: Engine Parsing ---
|
||||
|
||||
@@ -222,6 +284,27 @@ def test_nccl_receive_weights_without_init_raises():
|
||||
engine.receive_weights(update_info, lambda x: None)
|
||||
|
||||
|
||||
def test_nccl_receive_sparse_weights_without_init_raises():
|
||||
"""Test that sparse receive raises if init_transfer_engine wasn't called."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10]],
|
||||
num_updates_list=[2],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
engine.receive_sparse_weights(update_info, lambda x: None)
|
||||
|
||||
|
||||
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
|
||||
|
||||
|
||||
@@ -379,6 +462,136 @@ def test_nccl_weight_transfer_between_processes():
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_sparse_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
) -> bool:
|
||||
"""Trainer task that broadcasts sparse patches via NCCL."""
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerSendWeightsArgs,
|
||||
NCCLWeightTransferEngine,
|
||||
)
|
||||
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address,
|
||||
port=master_port,
|
||||
rank=0,
|
||||
world_size=world_size,
|
||||
)
|
||||
comm = PyNcclCommunicator(pg, device=0)
|
||||
|
||||
patch = SparseWeightPatch(
|
||||
name="test.weight",
|
||||
indices=torch.tensor([1, 7, 25], dtype=torch.int32, device="cuda:0"),
|
||||
values=torch.tensor([10.0, 20.0, 30.0], dtype=torch.float32, device="cuda:0"),
|
||||
)
|
||||
NCCLWeightTransferEngine.trainer_send_sparse_weights(
|
||||
iter([patch]),
|
||||
NCCLTrainerSendWeightsArgs(group=comm),
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
return True
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def inference_receive_sparse_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
) -> dict:
|
||||
"""Inference task that receives sparse patches via NCCLWeightTransferEngine."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
engine.init_transfer_engine(
|
||||
NCCLWeightTransferInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1,
|
||||
world_size=world_size,
|
||||
)
|
||||
)
|
||||
|
||||
target = torch.zeros(30, dtype=torch.float32, device="cuda")
|
||||
|
||||
def apply_sparse_patches(patches: list[SparseWeightPatch]):
|
||||
for patch in patches:
|
||||
target.index_copy_(0, patch.indices.to(torch.long), patch.values)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[30]],
|
||||
num_updates_list=[3],
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
engine.receive_sparse_weights(update_info, apply_sparse_patches)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
expected = torch.zeros(30, dtype=torch.float32, device="cuda")
|
||||
expected[[1, 7, 25]] = torch.tensor(
|
||||
[10.0, 20.0, 30.0], dtype=torch.float32, device="cuda"
|
||||
)
|
||||
success = torch.equal(target, expected)
|
||||
engine.shutdown()
|
||||
return {
|
||||
"success": success,
|
||||
"selected_values": target[[1, 7, 25]].cpu().tolist(),
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL sparse weight transfer test.",
|
||||
)
|
||||
def test_nccl_sparse_weight_transfer_between_processes():
|
||||
"""Test NCCL sparse weight transfer from trainer to inference process."""
|
||||
ray.init(ignore_reinit_error=True)
|
||||
|
||||
master_address = "127.0.0.1"
|
||||
master_port = get_open_port()
|
||||
world_size = 2
|
||||
|
||||
inference_future = inference_receive_sparse_tensor.remote(
|
||||
master_address, master_port, world_size
|
||||
)
|
||||
trainer_future = trainer_broadcast_sparse_tensor.remote(
|
||||
master_address, master_port, world_size
|
||||
)
|
||||
|
||||
trainer_result, result = ray.get([trainer_future, inference_future])
|
||||
|
||||
assert trainer_result, "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
"Sparse weight transfer failed. "
|
||||
f"Received selected values: {result['selected_values']}"
|
||||
)
|
||||
|
||||
|
||||
# --- Unit Tests: IPCWeightTransferUpdateInfo Validation ---
|
||||
|
||||
|
||||
@@ -461,9 +674,101 @@ class TestIPCWeightTransferUpdateInfoValidation:
|
||||
ipc_handles=ipc_handles,
|
||||
)
|
||||
|
||||
def test_missing_ipc_handles_raises(self):
|
||||
"""Test that omitting ipc_handles raises TypeError."""
|
||||
with pytest.raises(TypeError):
|
||||
def test_sparse_update_kind_rejected(self):
|
||||
"""Test that IPC backend rejects sparse update metadata."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
ipc_handle = reduce_tensor(dummy_tensor)
|
||||
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
|
||||
ipc_handles = [{gpu_uuid: ipc_handle}]
|
||||
|
||||
with pytest.raises(NotImplementedError, match="dense updates"):
|
||||
IPCWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
num_updates_list=[1],
|
||||
ipc_handles=ipc_handles,
|
||||
update_kind="sparse_flat",
|
||||
)
|
||||
|
||||
def test_sparse_methods_not_supported(self):
|
||||
"""Test that IPC engine inherits sparse rejection from the base class."""
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
|
||||
engine.receive_sparse_weights(MagicMock(), lambda _: None)
|
||||
with pytest.raises(NotImplementedError, match="(?i)sparse weight updates"):
|
||||
engine.trainer_send_sparse_weights(
|
||||
iter([]),
|
||||
{"mode": "http", "url": "http://localhost:8000"},
|
||||
)
|
||||
|
||||
def test_valid_update_info_from_pickled(self, monkeypatch):
|
||||
"""Test creating IPCWeightTransferUpdateInfo from pickled handles."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
|
||||
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
ipc_handle = reduce_tensor(dummy_tensor)
|
||||
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
|
||||
ipc_handles = [{gpu_uuid: ipc_handle}]
|
||||
|
||||
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
|
||||
|
||||
info = IPCWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
ipc_handles_pickled=pickled,
|
||||
)
|
||||
assert info.ipc_handles == ipc_handles
|
||||
assert info.ipc_handles_pickled is None
|
||||
|
||||
def test_pickled_requires_insecure_serialization_flag(self, monkeypatch):
|
||||
"""Test that pickled handles are rejected unless env flag is enabled."""
|
||||
monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "0")
|
||||
|
||||
with pytest.raises(ValueError, match="VLLM_ALLOW_INSECURE_SERIALIZATION=1"):
|
||||
IPCWeightTransferUpdateInfo(
|
||||
names=[],
|
||||
dtype_names=[],
|
||||
shapes=[],
|
||||
ipc_handles_pickled=base64.b64encode(pickle.dumps([])).decode("utf-8"),
|
||||
)
|
||||
|
||||
def test_both_handles_and_pickled_raises(self):
|
||||
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
dummy_tensor = torch.ones(10, 10, device="cuda:0")
|
||||
ipc_handle = reduce_tensor(dummy_tensor)
|
||||
gpu_uuid = str(torch.cuda.get_device_properties(0).uuid)
|
||||
ipc_handles = [{gpu_uuid: ipc_handle}]
|
||||
|
||||
pickled = base64.b64encode(pickle.dumps(ipc_handles)).decode("utf-8")
|
||||
|
||||
with pytest.raises(ValueError, match="Cannot specify both"):
|
||||
IPCWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
ipc_handles=ipc_handles,
|
||||
ipc_handles_pickled=pickled,
|
||||
)
|
||||
|
||||
def test_neither_handles_nor_pickled_raises(self):
|
||||
"""Test that providing neither ipc_handles nor ipc_handles_pickled raises."""
|
||||
with pytest.raises(ValueError, match="must be provided"):
|
||||
IPCWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
@@ -558,6 +863,28 @@ class TestIPCEngineParsing:
|
||||
assert gpu_uuid in update_info.ipc_handles[0]
|
||||
assert gpu_uuid in update_info.ipc_handles[1]
|
||||
|
||||
def test_parse_update_info_ignores_none_pickled_handles(self):
|
||||
"""Test Ray/asdict payloads with a null pickled field use ipc_handles."""
|
||||
config = WeightTransferConfig(backend="ipc")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = IPCWeightTransferEngine(
|
||||
config, parallel_config, MagicMock(spec=torch.nn.Module)
|
||||
)
|
||||
ipc_handles = [{"gpu-uuid": ("ipc-args",)}]
|
||||
|
||||
update_info = engine.parse_update_info(
|
||||
{
|
||||
"names": ["w1"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[1]],
|
||||
"ipc_handles": ipc_handles,
|
||||
"ipc_handles_pickled": None,
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(update_info, IPCWeightTransferUpdateInfo)
|
||||
assert update_info.ipc_handles == ipc_handles
|
||||
|
||||
def test_parse_update_info_both_handles_and_pickled_raises(self):
|
||||
"""Test that providing both ipc_handles and ipc_handles_pickled raises."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
|
||||
@@ -48,6 +48,7 @@ class MockUpdateInfo(WeightTransferUpdateInfo):
|
||||
names: list[str] | None = None
|
||||
dtype_names: list[str] | None = None
|
||||
shapes: list[list[int]] | None = None
|
||||
num_updates_list: list[int] | None = None
|
||||
|
||||
|
||||
class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo]):
|
||||
@@ -87,6 +88,15 @@ class MockWeightTransferEngine(WeightTransferEngine[MockInitInfo, MockUpdateInfo
|
||||
# (In real implementation, this would receive and load actual weights)
|
||||
load_weights([])
|
||||
|
||||
def receive_sparse_weights(
|
||||
self,
|
||||
update_info: MockUpdateInfo,
|
||||
apply_patches: Callable[[list], None],
|
||||
) -> None:
|
||||
MockWeightTransferEngine.receive_weights_called = True
|
||||
MockWeightTransferEngine.last_update_info = update_info
|
||||
apply_patches([])
|
||||
|
||||
def shutdown(self) -> None:
|
||||
MockWeightTransferEngine.shutdown_called = True
|
||||
|
||||
@@ -198,8 +208,6 @@ def test_update_weights_calls_engine():
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "init"})
|
||||
)
|
||||
|
||||
# Start weight update (required before update_weights)
|
||||
llm.start_weight_update(is_checkpoint_format=True)
|
||||
|
||||
# Call update_weights
|
||||
@@ -232,14 +240,67 @@ def test_update_weights_calls_engine():
|
||||
assert dtypes == test_dtypes
|
||||
assert shapes == test_shapes
|
||||
|
||||
# Finish weight update
|
||||
llm.finish_weight_update()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_update_weights_passes_sparse_metadata():
|
||||
"""Test sparse update metadata is forwarded unchanged to the engine."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
os.environ["VLLM_ENABLE_V1_MULTIPROCESSING"] = "0"
|
||||
os.environ["VLLM_ALLOW_INSECURE_SERIALIZATION"] = "1"
|
||||
|
||||
with patch(
|
||||
"vllm.v1.worker.gpu_worker.WeightTransferEngineFactory.create_engine",
|
||||
mock_create_engine,
|
||||
):
|
||||
llm = LLM(
|
||||
model=MODEL_NAME,
|
||||
enforce_eager=True,
|
||||
load_format="dummy",
|
||||
tensor_parallel_size=1,
|
||||
weight_transfer_config=WeightTransferConfig(backend="nccl"),
|
||||
)
|
||||
|
||||
llm.init_weight_transfer_engine(
|
||||
WeightTransferInitRequest(init_info={"test_param": "init"})
|
||||
)
|
||||
llm.start_weight_update(is_checkpoint_format=False)
|
||||
|
||||
llm.update_weights(
|
||||
WeightTransferUpdateRequest(
|
||||
update_info={
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["bfloat16"],
|
||||
"shapes": [[100]],
|
||||
"num_updates_list": [3],
|
||||
"update_kind": "sparse_flat",
|
||||
}
|
||||
)
|
||||
)
|
||||
|
||||
def check_sparse_update_called(self):
|
||||
engine = self.weight_transfer_engine
|
||||
if not engine.receive_weights_called:
|
||||
return None
|
||||
info = engine.last_update_info
|
||||
return (
|
||||
info.update_kind,
|
||||
info.num_updates_list,
|
||||
)
|
||||
|
||||
results = llm.collective_rpc(check_sparse_update_called)
|
||||
for result in results:
|
||||
assert result == ("sparse_flat", [3])
|
||||
|
||||
llm.finish_weight_update()
|
||||
|
||||
|
||||
@create_new_process_for_each_test()
|
||||
def test_full_weight_transfer_flow():
|
||||
"""Test the complete weight transfer flow:
|
||||
init -> start -> update -> finish."""
|
||||
"""Test the complete weight transfer flow: init -> start -> update -> finish."""
|
||||
if torch.accelerator.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ from unittest.mock import Mock
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
import vllm.v1.worker.gpu_model_runner as gpu_model_runner_module
|
||||
from vllm.config import (
|
||||
@@ -22,6 +23,7 @@ from vllm.distributed.parallel_state import (
|
||||
init_distributed_environment,
|
||||
initialize_model_parallel,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.model_executor.layers.mamba.mamba_mixer2 import MambaMixer2
|
||||
from vllm.platforms import current_platform
|
||||
@@ -784,6 +786,73 @@ def test_sample_passes_reordered_draft_probs_to_rejection_sampler():
|
||||
assert torch.equal(passed_draft_probs, expected_draft_probs)
|
||||
|
||||
|
||||
def test_apply_sparse_weight_patches_updates_only_selected_entries():
|
||||
class DummyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(6, dtype=torch.float32))
|
||||
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.model = DummyModel()
|
||||
|
||||
runner.apply_sparse_weight_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="weight",
|
||||
indices=torch.tensor([1, 4], dtype=torch.int32),
|
||||
values=torch.tensor([3.5, -2.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
expected = torch.tensor([0.0, 3.5, 0.0, 0.0, -2.0, 0.0], dtype=torch.float32)
|
||||
assert torch.equal(runner.get_model().weight.data, expected)
|
||||
|
||||
|
||||
def test_apply_sparse_weight_patches_rejects_mismatched_lengths():
|
||||
class DummyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.zeros(4, dtype=torch.float32))
|
||||
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.model = DummyModel()
|
||||
|
||||
with pytest.raises(ValueError, match="matching lengths"):
|
||||
runner.apply_sparse_weight_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="weight",
|
||||
indices=torch.tensor([1, 2], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_apply_sparse_weight_patches_rejects_non_contiguous_param():
|
||||
class DummyModel(nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(
|
||||
torch.arange(12, dtype=torch.float32).view(3, 4).t()
|
||||
)
|
||||
|
||||
runner = object.__new__(GPUModelRunner)
|
||||
runner.model = DummyModel()
|
||||
|
||||
with pytest.raises(NotImplementedError, match="contiguous params"):
|
||||
runner.apply_sparse_weight_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="weight",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([1.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def test_init_kv_cache_with_kv_sharing_invalid_target_layer_order(default_vllm_config):
|
||||
torch.set_default_dtype(torch.float16)
|
||||
layer_0 = "model.layers.0.self_attn.attn"
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.base import SparseWeightPatch
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferEngine
|
||||
from vllm.v1.worker.gpu_worker import Worker
|
||||
|
||||
|
||||
def _make_nccl_engine() -> NCCLWeightTransferEngine:
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
return NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
parallel_config,
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
|
||||
|
||||
def test_update_weights_sparse_dispatches_to_sparse_receive(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=1)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = False
|
||||
|
||||
applied_patches = []
|
||||
|
||||
def apply_sparse_weight_patches(patches):
|
||||
applied_patches.extend(patches)
|
||||
|
||||
worker.model_runner = SimpleNamespace(
|
||||
apply_sparse_weight_patches=apply_sparse_weight_patches,
|
||||
)
|
||||
|
||||
received_kinds = []
|
||||
|
||||
def receive_sparse_weights(update_info, apply_patches):
|
||||
received_kinds.append(update_info.update_kind)
|
||||
apply_patches(
|
||||
[
|
||||
SparseWeightPatch(
|
||||
name="layer.weight",
|
||||
indices=torch.tensor([1], dtype=torch.int32),
|
||||
values=torch.tensor([2.0], dtype=torch.float32),
|
||||
)
|
||||
]
|
||||
)
|
||||
|
||||
worker.weight_transfer_engine.receive_sparse_weights = receive_sparse_weights
|
||||
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[4]],
|
||||
"num_updates_list": [1],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
|
||||
assert received_kinds == ["sparse_flat"]
|
||||
assert len(applied_patches) == 1
|
||||
assert torch.equal(applied_patches[0].indices, torch.tensor([1], dtype=torch.int32))
|
||||
|
||||
|
||||
def test_update_weights_sparse_rejects_tp_or_pp(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=2)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = False
|
||||
worker.model_runner = SimpleNamespace(apply_sparse_weight_patches=lambda _: None)
|
||||
|
||||
with pytest.raises(NotImplementedError, match="TP=1 and PP=1"):
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[4]],
|
||||
"num_updates_list": [1],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
assert worker._weight_update_active is False
|
||||
assert worker._is_checkpoint_format is True
|
||||
|
||||
|
||||
def test_update_weights_sparse_rejects_checkpoint_format(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=1)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = True
|
||||
worker.model_runner = SimpleNamespace(model=MagicMock())
|
||||
|
||||
with pytest.raises(ValueError, match="start_weight_update"):
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": ["layer.weight"],
|
||||
"dtype_names": ["float32"],
|
||||
"shapes": [[4]],
|
||||
"num_updates_list": [1],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
assert worker._weight_update_active is False
|
||||
assert worker._is_checkpoint_format is True
|
||||
|
||||
|
||||
def test_update_weights_resets_state_when_update_info_is_invalid(monkeypatch):
|
||||
monkeypatch.setattr(torch.accelerator, "synchronize", lambda: None)
|
||||
|
||||
worker = object.__new__(Worker)
|
||||
worker.device = "cpu"
|
||||
worker.parallel_config = SimpleNamespace(world_size=1)
|
||||
worker.weight_transfer_engine = _make_nccl_engine()
|
||||
worker._weight_update_active = True
|
||||
worker._is_checkpoint_format = False
|
||||
|
||||
with pytest.raises(ValueError, match="cannot be empty"):
|
||||
Worker.update_weights(
|
||||
worker,
|
||||
{
|
||||
"names": [],
|
||||
"dtype_names": [],
|
||||
"shapes": [],
|
||||
"num_updates_list": [],
|
||||
"update_kind": "sparse_flat",
|
||||
},
|
||||
)
|
||||
assert worker._weight_update_active is False
|
||||
assert worker._is_checkpoint_format is True
|
||||
Reference in New Issue
Block a user