[Feat][RL][1/2] Native Weight Syncing API: NCCL (#31943)
Signed-off-by: ahao-anyscale <ahao@anyscale.com> Signed-off-by: Aaron Hao <ahao@anyscale.com> Co-authored-by: SumanthRH <sumanthrh99@gmail.com>
This commit is contained in:
@@ -0,0 +1,443 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for packed tensor broadcasting functionality.
|
||||
|
||||
Unit tests for packed_broadcast_producer and packed_broadcast_consumer.
|
||||
These utilities enable efficient batched tensor transfer over NCCL.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferUpdateInfo
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
packed_broadcast_consumer,
|
||||
packed_broadcast_producer,
|
||||
)
|
||||
|
||||
|
||||
class MockCommunicationGroup:
|
||||
"""Mock communication group for testing producer broadcast operations."""
|
||||
|
||||
def __init__(self):
|
||||
self.broadcasted_tensors: list[torch.Tensor] = []
|
||||
self.broadcast_count = 0
|
||||
self.device = torch.device("cuda:0")
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
"""Mock broadcast that stores the tensor for later verification."""
|
||||
self.broadcasted_tensors.append(tensor.clone())
|
||||
self.broadcast_count += 1
|
||||
|
||||
|
||||
class MockConsumerCommunicationGroup:
|
||||
"""Mock communication group for consumer that returns pre-stored tensors."""
|
||||
|
||||
def __init__(self, tensors_to_return: list[torch.Tensor]):
|
||||
self.tensors_to_return = tensors_to_return
|
||||
self.current_index = 0
|
||||
self.device = torch.device("cuda:0")
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
"""Mock broadcast that fills the tensor with pre-stored data."""
|
||||
if self.current_index < len(self.tensors_to_return):
|
||||
tensor.copy_(self.tensors_to_return[self.current_index])
|
||||
self.current_index += 1
|
||||
|
||||
|
||||
def create_mock_model_params(
|
||||
num_layers: int = 3,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Create mock model parameters for testing."""
|
||||
params = []
|
||||
for i in range(num_layers):
|
||||
params.append((f"layer{i}.weight", torch.randn(10, 20, dtype=dtype)))
|
||||
params.append((f"layer{i}.bias", torch.randn(10, dtype=dtype)))
|
||||
return params
|
||||
|
||||
|
||||
def create_state_dict_info(
|
||||
params: list[tuple[str, torch.Tensor]],
|
||||
) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
|
||||
"""Create state dict info (name -> (shape, dtype)) from params."""
|
||||
return {name: (tuple(tensor.shape), tensor.dtype) for name, tensor in params}
|
||||
|
||||
|
||||
# --- Unit Tests: NCCLWeightTransferUpdateInfo packed field ---
|
||||
|
||||
|
||||
class TestNCCLWeightTransferUpdateInfoPacked:
|
||||
"""Test NCCLWeightTransferUpdateInfo dataclass packed field."""
|
||||
|
||||
def test_packed_default_false(self):
|
||||
"""Test that packed defaults to False."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
)
|
||||
assert info.packed is False
|
||||
|
||||
def test_packed_can_be_set_true(self):
|
||||
"""Test that packed can be set to True."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10, 10]],
|
||||
packed=True,
|
||||
)
|
||||
assert info.packed is True
|
||||
|
||||
|
||||
# --- Unit Tests: packed_broadcast_producer ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastProducer:
|
||||
"""Test packed_broadcast_producer function."""
|
||||
|
||||
def test_producer_broadcasts_tensors(self):
|
||||
"""Test that producer broadcasts all tensors."""
|
||||
params = create_mock_model_params()
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
mock_group = MockCommunicationGroup()
|
||||
|
||||
# Use a small target size to force multiple batches
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=mock_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=500,
|
||||
)
|
||||
|
||||
# Should have broadcasted some tensors
|
||||
assert mock_group.broadcast_count > 0
|
||||
assert len(mock_group.broadcasted_tensors) > 0
|
||||
|
||||
def test_producer_single_large_tensor(self):
|
||||
"""Test with a single tensor larger than target size."""
|
||||
# Create a large tensor
|
||||
large_tensor = torch.randn(1000, 1000, dtype=torch.float32).cuda()
|
||||
params = [("large_weight", large_tensor)]
|
||||
|
||||
mock_group = MockCommunicationGroup()
|
||||
|
||||
# Small target size to force the tensor to exceed it
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=mock_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=100,
|
||||
)
|
||||
|
||||
# Should still broadcast the tensor (at least 1 broadcast)
|
||||
assert mock_group.broadcast_count >= 1
|
||||
assert len(mock_group.broadcasted_tensors) >= 1
|
||||
|
||||
# Verify the total broadcasted size matches the tensor
|
||||
expected_size = large_tensor.numel() * large_tensor.element_size()
|
||||
actual_size = sum(t.numel() for t in mock_group.broadcasted_tensors)
|
||||
assert actual_size == expected_size
|
||||
|
||||
def test_producer_multiple_batches(self):
|
||||
"""Test that tensors are properly batched when exceeding target size."""
|
||||
# Create many small tensors
|
||||
params = [
|
||||
(f"weight_{i}", torch.randn(10, 10, dtype=torch.float32).cuda())
|
||||
for i in range(20)
|
||||
]
|
||||
|
||||
mock_group = MockCommunicationGroup()
|
||||
|
||||
# Small target size to force multiple batches
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=mock_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=2000,
|
||||
)
|
||||
|
||||
# Should have multiple broadcasts
|
||||
assert mock_group.broadcast_count > 1
|
||||
|
||||
# Total size should match sum of all tensors
|
||||
expected_total = sum(t.numel() * t.element_size() for _, t in params)
|
||||
actual_total = sum(t.numel() for t in mock_group.broadcasted_tensors)
|
||||
assert actual_total == expected_total
|
||||
|
||||
def test_producer_empty_iterator(self):
|
||||
"""Test producer handles empty iterator gracefully."""
|
||||
mock_group = MockCommunicationGroup()
|
||||
|
||||
packed_broadcast_producer(
|
||||
iterator=iter([]),
|
||||
group=mock_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
|
||||
# No broadcasts for empty iterator
|
||||
assert mock_group.broadcast_count == 0
|
||||
|
||||
|
||||
# --- Unit Tests: packed_broadcast_consumer ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastConsumer:
|
||||
"""Test packed_broadcast_consumer function."""
|
||||
|
||||
def test_consumer_receives_tensors(self):
|
||||
"""Test that consumer receives and unpacks tensors."""
|
||||
params = create_mock_model_params()
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
buffer_size = 2000
|
||||
|
||||
# First, run producer to get the broadcasted tensors
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Now run consumer with the broadcasted tensors
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all parameters were unpacked
|
||||
assert len(unpacked_tensors) == len(params)
|
||||
|
||||
# Verify each tensor matches the original
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-5, atol=1e-7)
|
||||
|
||||
|
||||
# --- Integration Tests: Producer-Consumer Roundtrip ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastRoundtrip:
|
||||
"""Test producer-consumer roundtrip behavior."""
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_roundtrip_different_dtypes(self, dtype):
|
||||
"""Test roundtrip with different data types."""
|
||||
params = create_mock_model_params(num_layers=2, dtype=dtype)
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
buffer_size = 1000
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify roundtrip preserves data
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.dtype == dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_roundtrip_mixed_dtypes(self):
|
||||
"""Test roundtrip with mixed data types."""
|
||||
# Create params with mixed dtypes
|
||||
params = [
|
||||
("layer1.weight", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("layer1.bias", torch.randn(10, dtype=torch.float16).cuda()),
|
||||
("layer2.weight", torch.randn(20, 30, dtype=torch.bfloat16).cuda()),
|
||||
]
|
||||
|
||||
buffer_size = 500
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all params roundtrip correctly with correct dtypes
|
||||
for name, original_tensor in params:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
@pytest.mark.parametrize("target_size", [100, 1000, 10000, 100000])
|
||||
def test_roundtrip_different_batch_sizes(self, target_size):
|
||||
"""Test roundtrip with different target batch sizes."""
|
||||
params = create_mock_model_params(num_layers=5)
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=target_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=target_size,
|
||||
)
|
||||
|
||||
# Verify all params roundtrip correctly
|
||||
assert len(unpacked_tensors) == len(params)
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
assert torch.allclose(
|
||||
unpacked_tensors[name], original_tensor, rtol=1e-5, atol=1e-7
|
||||
)
|
||||
|
||||
def test_roundtrip_non_contiguous_tensors(self):
|
||||
"""Test roundtrip with non-contiguous tensors from the trainer."""
|
||||
# Create non-contiguous tensors (simulating trainer outputs)
|
||||
# Transposed tensors are non-contiguous
|
||||
weight1 = torch.randn(20, 10, dtype=torch.float32).cuda().T
|
||||
# Sliced tensors with step are non-contiguous
|
||||
weight2 = torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]
|
||||
# Permuted tensors are non-contiguous
|
||||
weight3 = torch.randn(5, 10, 15, dtype=torch.bfloat16).cuda().permute(2, 0, 1)
|
||||
|
||||
params = [
|
||||
("layer1.weight", weight1),
|
||||
("layer2.weight", weight2),
|
||||
("layer3.weight", weight3),
|
||||
]
|
||||
|
||||
# Verify tensors are indeed non-contiguous
|
||||
for name, tensor in params:
|
||||
assert not tensor.is_contiguous(), f"{name} should be non-contiguous"
|
||||
|
||||
buffer_size = 500
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all non-contiguous params roundtrip correctly
|
||||
for name, original_tensor in params:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
@@ -0,0 +1,346 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for weight transfer engine backends.
|
||||
|
||||
Unit tests for engine classes (parsing, validation, registry).
|
||||
Integration test for NCCL weight transfer between processes using Ray.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer import WeightTransferEngineFactory
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
from vllm.utils.network_utils import get_open_port
|
||||
|
||||
|
||||
def create_mock_parallel_config(
|
||||
rank: int = 0,
|
||||
world_size: int = 1,
|
||||
dp_rank: int = 0,
|
||||
) -> ParallelConfig:
|
||||
"""Create a mock ParallelConfig for testing."""
|
||||
config = MagicMock(spec=ParallelConfig)
|
||||
config.rank = rank
|
||||
config.world_size = world_size
|
||||
config.data_parallel_rank = dp_rank
|
||||
return config
|
||||
|
||||
|
||||
# --- Unit Tests: NCCLWeightTransferUpdateInfo Validation ---
|
||||
|
||||
|
||||
class TestNCCLWeightTransferUpdateInfoValidation:
|
||||
"""Test NCCLWeightTransferUpdateInfo dataclass validation."""
|
||||
|
||||
def test_valid_update_info(self):
|
||||
"""Test creating valid NCCLWeightTransferUpdateInfo."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "float32"],
|
||||
shapes=[[10, 10], [10]],
|
||||
)
|
||||
assert info.names == ["layer.weight", "layer.bias"]
|
||||
assert info.dtype_names == ["float32", "float32"]
|
||||
assert info.shapes == [[10, 10], [10]]
|
||||
|
||||
def test_mismatched_dtype_names_raises(self):
|
||||
"""Test that mismatched dtype_names length raises ValueError."""
|
||||
with pytest.raises(ValueError, match="dtype_names"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32"], # Only one dtype
|
||||
shapes=[[10, 10], [10]],
|
||||
)
|
||||
|
||||
def test_mismatched_shapes_raises(self):
|
||||
"""Test that mismatched shapes length raises ValueError."""
|
||||
with pytest.raises(ValueError, match="shapes"):
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["layer.weight", "layer.bias"],
|
||||
dtype_names=["float32", "float32"],
|
||||
shapes=[[10, 10]], # Only one shape
|
||||
)
|
||||
|
||||
def test_empty_lists_valid(self):
|
||||
"""Test that empty lists are valid."""
|
||||
info = NCCLWeightTransferUpdateInfo(
|
||||
names=[],
|
||||
dtype_names=[],
|
||||
shapes=[],
|
||||
)
|
||||
assert len(info.names) == 0
|
||||
|
||||
|
||||
# --- Unit Tests: Engine Parsing ---
|
||||
|
||||
|
||||
class TestNCCLEngineParsing:
|
||||
"""Test NCCLWeightTransferEngine parsing methods."""
|
||||
|
||||
def test_parse_init_info_valid(self):
|
||||
"""Test parsing valid init info dict."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
|
||||
init_info = engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
"master_port": 12345,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(init_info, NCCLWeightTransferInitInfo)
|
||||
assert init_info.master_address == "127.0.0.1"
|
||||
assert init_info.master_port == 12345
|
||||
assert init_info.rank_offset == 1
|
||||
assert init_info.world_size == 3
|
||||
|
||||
def test_parse_init_info_missing_field_raises(self):
|
||||
"""Test parsing init info with missing required field."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid init_info"):
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
# Missing master_port, rank_offset, world_size
|
||||
}
|
||||
)
|
||||
|
||||
def test_parse_update_info_valid(self):
|
||||
"""Test parsing valid update info dict."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
|
||||
update_info = engine.parse_update_info(
|
||||
{
|
||||
"names": ["w1", "w2"],
|
||||
"dtype_names": ["float32", "bfloat16"],
|
||||
"shapes": [[100, 100], [50]],
|
||||
}
|
||||
)
|
||||
|
||||
assert isinstance(update_info, NCCLWeightTransferUpdateInfo)
|
||||
assert update_info.names == ["w1", "w2"]
|
||||
assert update_info.dtype_names == ["float32", "bfloat16"]
|
||||
assert update_info.shapes == [[100, 100], [50]]
|
||||
|
||||
|
||||
# --- Unit Tests: Engine Registry ---
|
||||
|
||||
|
||||
class TestEngineRegistry:
|
||||
"""Test weight transfer engine registry."""
|
||||
|
||||
def test_create_engine_nccl(self):
|
||||
"""Test factory creates NCCL engine."""
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = WeightTransferEngineFactory.create_engine(config, parallel_config)
|
||||
assert isinstance(engine, NCCLWeightTransferEngine)
|
||||
|
||||
def test_create_engine_invalid_backend(self):
|
||||
"""Test factory raises for invalid backend."""
|
||||
config = WeightTransferConfig(backend="invalid")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
with pytest.raises(ValueError, match="Invalid weight transfer backend"):
|
||||
WeightTransferEngineFactory.create_engine(config, parallel_config)
|
||||
|
||||
def test_register_duplicate_raises(self):
|
||||
"""Test registering duplicate engine name raises."""
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
WeightTransferEngineFactory.register_engine(
|
||||
"nccl", NCCLWeightTransferEngine
|
||||
)
|
||||
|
||||
|
||||
# --- Test receive_weights without init raises ---
|
||||
|
||||
|
||||
def test_nccl_receive_weights_without_init_raises():
|
||||
"""Test that receive_weights raises if init_transfer_engine wasn't called."""
|
||||
if torch.cuda.device_count() < 1:
|
||||
pytest.skip("Need at least 1 GPU for this test")
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = create_mock_parallel_config()
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["w"],
|
||||
dtype_names=["float32"],
|
||||
shapes=[[10]],
|
||||
)
|
||||
|
||||
with pytest.raises(RuntimeError, match="not initialized"):
|
||||
engine.receive_weights(update_info, lambda x: None)
|
||||
|
||||
|
||||
# --- Integration Test: NCCL Weight Transfer Between Ray Tasks ---
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> bool:
|
||||
"""Trainer task that broadcasts a tensor via NCCL."""
|
||||
import torch
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
|
||||
# Create process group as rank 0 (trainer)
|
||||
pg = StatelessProcessGroup.create(
|
||||
host=master_address,
|
||||
port=master_port,
|
||||
rank=0,
|
||||
world_size=world_size,
|
||||
)
|
||||
# Ray sets CUDA_VISIBLE_DEVICES, so device 0 is the assigned GPU
|
||||
comm = PyNcclCommunicator(pg, device=0)
|
||||
|
||||
# Create and broadcast the tensor
|
||||
dtype = getattr(torch, tensor_dtype)
|
||||
tensor_to_send = torch.ones(tensor_shape, dtype=dtype, device="cuda:0")
|
||||
comm.broadcast(tensor_to_send, src=0, stream=torch.cuda.current_stream())
|
||||
torch.cuda.synchronize()
|
||||
|
||||
return True
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def inference_receive_tensor(
|
||||
master_address: str,
|
||||
master_port: int,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Inference task that receives tensor via NCCLWeightTransferEngine."""
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
# Create engine with mock parallel config
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
|
||||
engine = NCCLWeightTransferEngine(config, parallel_config)
|
||||
|
||||
# Initialize the engine (joins as rank 1)
|
||||
init_info = NCCLWeightTransferInitInfo(
|
||||
master_address=master_address,
|
||||
master_port=master_port,
|
||||
rank_offset=1, # Trainer is rank 0, we become rank 1
|
||||
world_size=world_size,
|
||||
)
|
||||
engine.init_transfer_engine(init_info)
|
||||
|
||||
# Receive weights with a no-op load_weights that captures the tensor
|
||||
received_tensors = []
|
||||
|
||||
def noop_load_weights(weights: list[tuple[str, torch.Tensor]]):
|
||||
for name, tensor in weights:
|
||||
# Clone tensor to keep it after engine cleans up
|
||||
received_tensors.append((name, tensor.clone()))
|
||||
|
||||
update_info = NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=[tensor_dtype],
|
||||
shapes=[tensor_shape],
|
||||
)
|
||||
engine.receive_weights(update_info, noop_load_weights)
|
||||
torch.cuda.synchronize()
|
||||
|
||||
# Verify we received the tensor
|
||||
success = False
|
||||
received_shape = None
|
||||
received_sum = None
|
||||
|
||||
if len(received_tensors) == 1:
|
||||
name, tensor = received_tensors[0]
|
||||
received_shape = list(tensor.shape)
|
||||
received_sum = tensor.sum().item()
|
||||
# Check shape matches and values are all 1s (trainer sends ones)
|
||||
if received_shape == tensor_shape:
|
||||
expected_sum = 1.0 * torch.tensor(tensor_shape).prod().item()
|
||||
if abs(received_sum - expected_sum) < 0.01:
|
||||
success = True
|
||||
|
||||
engine.shutdown()
|
||||
|
||||
return {
|
||||
"success": success,
|
||||
"received_shape": received_shape,
|
||||
"received_sum": received_sum,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.cuda.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes():
|
||||
"""Test NCCL weight transfer from trainer to inference process using Ray.
|
||||
|
||||
This test verifies that the NCCLWeightTransferEngine can receive
|
||||
tensors broadcast by a trainer process via NCCL.
|
||||
"""
|
||||
ray.init(ignore_reinit_error=True)
|
||||
|
||||
master_address = "127.0.0.1"
|
||||
master_port = get_open_port()
|
||||
world_size = 2 # 1 trainer + 1 inference worker
|
||||
|
||||
# Tensor to transfer: 100x100 ones
|
||||
tensor_shape = [100, 100]
|
||||
tensor_dtype = "float32"
|
||||
|
||||
# Start both tasks concurrently - Ray assigns GPUs automatically
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
trainer_future = trainer_broadcast_tensor.remote(
|
||||
master_address, master_port, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
# Wait for both to complete
|
||||
trainer_result, result = ray.get([trainer_future, inference_future])
|
||||
|
||||
assert trainer_result, "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
f"Weight transfer failed. "
|
||||
f"Received shape: {result['received_shape']}, "
|
||||
f"Received sum: {result['received_sum']}"
|
||||
)
|
||||
Reference in New Issue
Block a user