[Feat][RL] IPC weight sync optimizations: multigpu support and chunked packed tensors (#37476)

Signed-off-by: ahao-anyscale <ahao@anyscale.com>
Signed-off-by: hao-aaron <ahao@anyscale.com>
This commit is contained in:
Aaron Hao
2026-05-15 22:53:06 +08:00
committed by GitHub
parent 0fe7550254
commit e0a45f1455
10 changed files with 1591 additions and 455 deletions
+478 -146
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for packed tensor broadcasting functionality.
Unit tests for packed_broadcast_producer and packed_broadcast_consumer.
Unit tests for packed_nccl_broadcast_producer and packed_nccl_broadcast_consumer.
These utilities enable efficient batched tensor transfer over NCCL.
"""
@@ -11,8 +11,12 @@ import torch
from vllm.distributed.weight_transfer.nccl_engine import NCCLWeightTransferUpdateInfo
from vllm.distributed.weight_transfer.packed_tensor import (
packed_broadcast_consumer,
packed_broadcast_producer,
pack_tensors,
packed_ipc_consumer,
packed_ipc_producer,
packed_nccl_broadcast_consumer,
packed_nccl_broadcast_producer,
unpack_tensor,
)
@@ -90,91 +94,18 @@ class TestNCCLWeightTransferUpdateInfoPacked:
assert info.packed is True
# --- Unit Tests: packed_broadcast_producer ---
# --- Unit Tests: packed_nccl_broadcast_producer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedBroadcastProducer:
"""Test packed_broadcast_producer function."""
def test_producer_broadcasts_tensors(self):
"""Test that producer broadcasts all tensors."""
params = create_mock_model_params()
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
mock_group = MockCommunicationGroup()
# Use a small target size to force multiple batches
packed_broadcast_producer(
iterator=iter(params_cuda),
group=mock_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=500,
)
# Should have broadcasted some tensors
assert mock_group.broadcast_count > 0
assert len(mock_group.broadcasted_tensors) > 0
def test_producer_single_large_tensor(self):
"""Test with a single tensor larger than target size."""
# Create a large tensor
large_tensor = torch.randn(1000, 1000, dtype=torch.float32).cuda()
params = [("large_weight", large_tensor)]
mock_group = MockCommunicationGroup()
# Small target size to force the tensor to exceed it
packed_broadcast_producer(
iterator=iter(params),
group=mock_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=100,
)
# Should still broadcast the tensor (at least 1 broadcast)
assert mock_group.broadcast_count >= 1
assert len(mock_group.broadcasted_tensors) >= 1
# Verify the total broadcasted size matches the tensor
expected_size = large_tensor.numel() * large_tensor.element_size()
actual_size = sum(t.numel() for t in mock_group.broadcasted_tensors)
assert actual_size == expected_size
def test_producer_multiple_batches(self):
"""Test that tensors are properly batched when exceeding target size."""
# Create many small tensors
params = [
(f"weight_{i}", torch.randn(10, 10, dtype=torch.float32).cuda())
for i in range(20)
]
mock_group = MockCommunicationGroup()
# Small target size to force multiple batches
packed_broadcast_producer(
iterator=iter(params),
group=mock_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=2000,
)
# Should have multiple broadcasts
assert mock_group.broadcast_count > 1
# Total size should match sum of all tensors
expected_total = sum(t.numel() * t.element_size() for _, t in params)
actual_total = sum(t.numel() for t in mock_group.broadcasted_tensors)
assert actual_total == expected_total
"""Test packed_nccl_broadcast_producer function."""
def test_producer_empty_iterator(self):
"""Test producer handles empty iterator gracefully."""
mock_group = MockCommunicationGroup()
packed_broadcast_producer(
packed_nccl_broadcast_producer(
iterator=iter([]),
group=mock_group,
src=0,
@@ -186,64 +117,6 @@ class TestPackedBroadcastProducer:
assert mock_group.broadcast_count == 0
# --- Unit Tests: packed_broadcast_consumer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedBroadcastConsumer:
"""Test packed_broadcast_consumer function."""
def test_consumer_receives_tensors(self):
"""Test that consumer receives and unpacks tensors."""
params = create_mock_model_params()
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
buffer_size = 2000
# First, run producer to get the broadcasted tensors
producer_group = MockCommunicationGroup()
packed_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
# Now run consumer with the broadcasted tensors
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params_cuda)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify all parameters were unpacked
assert len(unpacked_tensors) == len(params)
# Verify each tensor matches the original
for name, original_tensor in params_cuda:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.shape == original_tensor.shape
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-5, atol=1e-7)
# --- Integration Tests: Producer-Consumer Roundtrip ---
@@ -260,7 +133,7 @@ class TestPackedBroadcastRoundtrip:
buffer_size = 1000
producer_group = MockCommunicationGroup()
packed_broadcast_producer(
packed_nccl_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
@@ -279,7 +152,7 @@ class TestPackedBroadcastRoundtrip:
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_broadcast_consumer(
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
@@ -306,7 +179,7 @@ class TestPackedBroadcastRoundtrip:
buffer_size = 500
producer_group = MockCommunicationGroup()
packed_broadcast_producer(
packed_nccl_broadcast_producer(
iterator=iter(params),
group=producer_group,
src=0,
@@ -325,7 +198,7 @@ class TestPackedBroadcastRoundtrip:
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_broadcast_consumer(
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
@@ -341,7 +214,7 @@ class TestPackedBroadcastRoundtrip:
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
@pytest.mark.parametrize("target_size", [100, 1000, 10000, 100000])
@pytest.mark.parametrize("target_size", [100, 100000])
def test_roundtrip_different_batch_sizes(self, target_size):
"""Test roundtrip with different target batch sizes."""
params = create_mock_model_params(num_layers=5)
@@ -349,7 +222,7 @@ class TestPackedBroadcastRoundtrip:
producer_group = MockCommunicationGroup()
packed_broadcast_producer(
packed_nccl_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
@@ -368,7 +241,7 @@ class TestPackedBroadcastRoundtrip:
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_broadcast_consumer(
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
@@ -407,7 +280,7 @@ class TestPackedBroadcastRoundtrip:
buffer_size = 500
producer_group = MockCommunicationGroup()
packed_broadcast_producer(
packed_nccl_broadcast_producer(
iterator=iter(params),
group=producer_group,
src=0,
@@ -426,7 +299,7 @@ class TestPackedBroadcastRoundtrip:
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_broadcast_consumer(
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
@@ -441,3 +314,462 @@ class TestPackedBroadcastRoundtrip:
assert unpacked.shape == original_tensor.shape
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
# --- Unit Tests: unpack_tensor ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestUnpackTensor:
"""Test the shared unpack_tensor function."""
def test_unpack_produces_independent_copies(self):
"""Verify unpacked tensors don't share memory with packed buffer."""
original = torch.randn(10, dtype=torch.float32).cuda()
packed = original.contiguous().view(torch.uint8).view(-1)
result = unpack_tensor(
packed,
names=["w"],
shapes=[[10]],
dtypes=[torch.float32],
tensor_sizes=[packed.numel()],
)
# Mutate the packed buffer
packed.zero_()
# Unpacked tensor should be unaffected
assert torch.allclose(result[0][1], original)
# --- Unit Tests: pack_tensors ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackTensors:
"""Test the shared pack_tensors function."""
def test_pack_basic(self):
"""Test packing a few tensors into one buffer."""
params = [
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
("w2", torch.randn(5, dtype=torch.float16).cuda()),
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
assert chunk is not None
assert len(chunk.names) == 2
assert chunk.names == ["w1", "w2"]
assert chunk.shapes == [[10, 20], [5]]
assert chunk.dtypes == [torch.float32, torch.float16]
assert chunk.packed_tensor.dtype == torch.uint8
def test_pack_respects_buffer_limit(self):
"""Test that packing stops when buffer_size_bytes is exceeded."""
params = [
(f"w{i}", torch.randn(100, 100, dtype=torch.float32).cuda())
for i in range(10)
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=50_000,
)
assert chunk is not None
assert len(chunk.names) < 10
def test_pack_empty_iterator(self):
"""Test that an empty iterator returns None."""
chunk = pack_tensors(
iterator=iter([]),
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
assert chunk is None
def test_pack_single_tensor_larger_than_buffer_warns(self):
"""Test that a tensor exceeding buffer_size_bytes emits a warning."""
big = torch.randn(1000, 1000, dtype=torch.float32).cuda()
params = [("big", big)]
import warnings
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=100,
)
assert chunk is not None
assert len(chunk.names) == 1
assert any("exceeds buffer_size_bytes" in str(wi.message) for wi in w)
def test_pack_unpack_roundtrip(self):
"""Test pack then unpack produces identical tensors."""
params = [
("a", torch.randn(8, 16, dtype=torch.float32).cuda()),
("b", torch.randn(4, dtype=torch.float16).cuda()),
("c", torch.randn(3, 5, 7, dtype=torch.bfloat16).cuda()),
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
assert chunk is not None
result = unpack_tensor(
chunk.packed_tensor,
chunk.names,
chunk.shapes,
chunk.dtypes,
chunk.tensor_sizes,
)
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor, rtol=1e-4, atol=1e-6)
def test_pack_multiple_chunks(self):
"""Test consuming an iterator across multiple pack_tensors calls."""
params = [
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
]
it = iter(params)
all_names = []
chunks = []
while True:
chunk = pack_tensors(it, lambda x: x[1], buffer_size_bytes=12_000)
if chunk is None:
break
chunks.append(chunk)
all_names.extend(chunk.names)
assert len(chunks) > 1
assert all_names == [f"w{i}" for i in range(6)]
# --- Unit Tests: packed_ipc_producer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedIpcProducer:
"""Test the packed_ipc_producer generator."""
def test_producer_yields_chunks(self):
"""Test that the producer yields PackedIpcChunk objects."""
params = [
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="test-uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=12_000,
)
)
assert len(chunks) > 1
def test_producer_ipc_handle_has_uuid(self):
"""Test that each chunk's ipc_handle is keyed by the given UUID."""
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="my-gpu-uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
assert "my-gpu-uuid" in chunks[0].ipc_handle
def test_producer_dtype_names_are_strings(self):
"""Test that dtype_names are string representations."""
params = [
("a", torch.randn(10, dtype=torch.float32).cuda()),
("b", torch.randn(10, dtype=torch.float16).cuda()),
]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
assert chunks[0].dtype_names == ["float32", "float16"]
def test_producer_empty_iterator(self):
"""Test producer with empty iterator yields nothing."""
chunks = list(
packed_ipc_producer(
iterator=iter([]),
gpu_uuid="uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
)
assert len(chunks) == 0
# --- Integration Tests: IPC Producer-Consumer Roundtrip ---
def _ipc_consumer_worker(cmd_q, ack_q, result_q, done_event, device_index):
"""Worker that consumes chunks streamed one at a time from the parent.
CUDA IPC requires the consumer to be in a separate process from the
producer. The producer reuses a single IPC buffer between chunks, so
the parent must wait for our ack (sent after we copy the chunk to
CPU) before advancing the producer.
"""
try:
torch.accelerator.set_device_index(device_index)
all_results = []
while True:
cd = cmd_q.get()
if cd is None:
break
result = packed_ipc_consumer(
ipc_handle=cd["ipc_handle"],
names=cd["names"],
shapes=cd["shapes"],
dtype_names=cd["dtype_names"],
tensor_sizes=cd["tensor_sizes"],
device_index=device_index,
)
# .cpu() forces a GPU→CPU copy off the shared IPC buffer, so
# the producer is free to overwrite it once we ack.
all_results.extend([(name, tensor.cpu()) for name, tensor in result])
del result
ack_q.put("ack")
result_q.put(("ok", all_results))
except Exception as e:
result_q.put(("error", str(e)))
# Keep the process alive until the parent has finished reading from
# the result queue — torch serializes CPU tensors via fd sharing,
# which requires this process's resource-sharer server to be running.
done_event.wait(timeout=60)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedIpcRoundtrip:
"""Test IPC producer-consumer roundtrip using real CUDA IPC.
These tests spawn a child process for the consumer because
rebuild_cuda_tensor requires a separate process from the one that
called reduce_tensor.
"""
def _get_gpu_uuid(self) -> str:
device_index = torch.cuda.current_device()
props = torch.cuda.get_device_properties(device_index)
return str(props.uuid)
def _run_roundtrip(self, chunk_iter, device_index, timeout=30):
"""Stream chunks through a child consumer one at a time.
``packed_ipc_producer`` reuses a single IPC buffer for every
chunk, so the producer must not be advanced until the consumer
has finished reading the current chunk. We enforce that with an
ack queue: the consumer puts ``"ack"`` after it has copied the
chunk to CPU, and only then do we pull the next chunk from the
generator.
Returns ``(num_chunks, results)``.
"""
import multiprocessing as mp
ctx = mp.get_context("spawn")
cmd_q = ctx.Queue()
ack_q = ctx.Queue()
result_q = ctx.Queue()
done_event = ctx.Event()
proc = ctx.Process(
target=_ipc_consumer_worker,
args=(cmd_q, ack_q, result_q, done_event, device_index),
)
proc.start()
num_chunks = 0
try:
for chunk in chunk_iter:
cmd_q.put(
{
"ipc_handle": chunk.ipc_handle,
"names": chunk.names,
"shapes": chunk.shapes,
"dtype_names": chunk.dtype_names,
"tensor_sizes": chunk.tensor_sizes,
}
)
if ack_q.get(timeout=timeout) != "ack":
raise RuntimeError("Consumer did not ack chunk")
num_chunks += 1
cmd_q.put(None)
status, payload = result_q.get(timeout=timeout)
finally:
done_event.set()
proc.join(timeout=10)
if proc.is_alive():
proc.kill()
if status == "error":
raise RuntimeError(f"Consumer process failed: {payload}")
# Reclaim IPC-shared memory now that the child has released it
torch.cuda.ipc_collect()
return num_chunks, payload
def test_roundtrip_basic(self):
"""Test basic IPC producer -> consumer roundtrip."""
params = [
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
("w2", torch.randn(5, dtype=torch.float16).cuda()),
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
num_chunks, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
assert num_chunks == 1
assert len(result) == 2
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_roundtrip_dtypes(self, dtype):
"""Test IPC roundtrip with different dtypes."""
params = create_mock_model_params(num_layers=2, dtype=dtype)
params_cuda = [(n, t.cuda()) for n, t in params]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
_, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params_cuda),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
assert len(result) == len(params_cuda)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(
params_cuda, result
):
assert orig_name == res_name
assert res_tensor.dtype == dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
def test_roundtrip_multiple_chunks(self):
"""Test IPC roundtrip across multiple chunks."""
params = [
(f"layer{i}.weight", torch.randn(100, 100, dtype=torch.float32).cuda())
for i in range(8)
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
num_chunks, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=50_000,
),
device_index,
)
assert num_chunks > 1
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-5, atol=1e-7)
def test_roundtrip_non_contiguous(self):
"""Test IPC roundtrip with non-contiguous tensors."""
params = [
("transposed", torch.randn(20, 10, dtype=torch.float32).cuda().T),
("sliced", torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]),
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
for _, t in params:
assert not t.is_contiguous()
_, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
def test_consumer_wrong_uuid_raises(self):
"""Test that consumer raises ValueError for unknown GPU UUID."""
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
gpu_uuid = self._get_gpu_uuid()
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
c = chunks[0]
fake_handle = {"fake-uuid-12345": c.ipc_handle[gpu_uuid]}
with pytest.raises(ValueError, match="IPC handle not found"):
packed_ipc_consumer(
ipc_handle=fake_handle,
names=c.names,
shapes=c.shapes,
dtype_names=c.dtype_names,
tensor_sizes=c.tensor_sizes,
device_index=torch.cuda.current_device(),
)