[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,775 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for packed tensor broadcasting functionality.
|
||||
|
||||
Unit tests for packed_nccl_broadcast_producer and packed_nccl_broadcast_consumer.
|
||||
These utilities enable efficient batched tensor transfer over NCCL.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm.distributed.weight_transfer.packed_tensor import (
|
||||
pack_tensors,
|
||||
packed_ipc_consumer,
|
||||
packed_ipc_producer,
|
||||
packed_nccl_broadcast_consumer,
|
||||
packed_nccl_broadcast_producer,
|
||||
unpack_tensor,
|
||||
)
|
||||
|
||||
|
||||
class MockCommunicationGroup:
|
||||
"""Mock communication group for testing producer broadcast operations."""
|
||||
|
||||
def __init__(self):
|
||||
self.broadcasted_tensors: list[torch.Tensor] = []
|
||||
self.broadcast_count = 0
|
||||
self.device = torch.device("cuda:0")
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
"""Mock broadcast that stores the tensor for later verification."""
|
||||
self.broadcasted_tensors.append(tensor.clone())
|
||||
self.broadcast_count += 1
|
||||
|
||||
|
||||
class MockConsumerCommunicationGroup:
|
||||
"""Mock communication group for consumer that returns pre-stored tensors."""
|
||||
|
||||
def __init__(self, tensors_to_return: list[torch.Tensor]):
|
||||
self.tensors_to_return = tensors_to_return
|
||||
self.current_index = 0
|
||||
self.device = torch.device("cuda:0")
|
||||
|
||||
def broadcast(self, tensor, src):
|
||||
"""Mock broadcast that fills the tensor with pre-stored data."""
|
||||
if self.current_index < len(self.tensors_to_return):
|
||||
tensor.copy_(self.tensors_to_return[self.current_index])
|
||||
self.current_index += 1
|
||||
|
||||
|
||||
def create_mock_model_params(
|
||||
num_layers: int = 3,
|
||||
dtype: torch.dtype = torch.float32,
|
||||
) -> list[tuple[str, torch.Tensor]]:
|
||||
"""Create mock model parameters for testing."""
|
||||
params = []
|
||||
for i in range(num_layers):
|
||||
params.append((f"layer{i}.weight", torch.randn(10, 20, dtype=dtype)))
|
||||
params.append((f"layer{i}.bias", torch.randn(10, dtype=dtype)))
|
||||
return params
|
||||
|
||||
|
||||
def create_state_dict_info(
|
||||
params: list[tuple[str, torch.Tensor]],
|
||||
) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
|
||||
"""Create state dict info (name -> (shape, dtype)) from params."""
|
||||
return {name: (tuple(tensor.shape), tensor.dtype) for name, tensor in params}
|
||||
|
||||
|
||||
# --- Unit Tests: packed_nccl_broadcast_producer ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastProducer:
|
||||
"""Test packed_nccl_broadcast_producer function."""
|
||||
|
||||
def test_producer_empty_iterator(self):
|
||||
"""Test producer handles empty iterator gracefully."""
|
||||
mock_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter([]),
|
||||
group=mock_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
|
||||
# No broadcasts for empty iterator
|
||||
assert mock_group.broadcast_count == 0
|
||||
|
||||
|
||||
# --- Integration Tests: Producer-Consumer Roundtrip ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedBroadcastRoundtrip:
|
||||
"""Test producer-consumer roundtrip behavior."""
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_roundtrip_different_dtypes(self, dtype):
|
||||
"""Test roundtrip with different data types."""
|
||||
params = create_mock_model_params(num_layers=2, dtype=dtype)
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
buffer_size = 1000
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify roundtrip preserves data
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.dtype == dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_roundtrip_mixed_dtypes(self):
|
||||
"""Test roundtrip with mixed data types."""
|
||||
# Create params with mixed dtypes
|
||||
params = [
|
||||
("layer1.weight", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("layer1.bias", torch.randn(10, dtype=torch.float16).cuda()),
|
||||
("layer2.weight", torch.randn(20, 30, dtype=torch.bfloat16).cuda()),
|
||||
]
|
||||
|
||||
buffer_size = 500
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all params roundtrip correctly with correct dtypes
|
||||
for name, original_tensor in params:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
@pytest.mark.parametrize("target_size", [100, 100000])
|
||||
def test_roundtrip_different_batch_sizes(self, target_size):
|
||||
"""Test roundtrip with different target batch sizes."""
|
||||
params = create_mock_model_params(num_layers=5)
|
||||
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
|
||||
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params_cuda),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=target_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params_cuda)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=target_size,
|
||||
)
|
||||
|
||||
# Verify all params roundtrip correctly
|
||||
assert len(unpacked_tensors) == len(params)
|
||||
for name, original_tensor in params_cuda:
|
||||
assert name in unpacked_tensors
|
||||
assert torch.allclose(
|
||||
unpacked_tensors[name], original_tensor, rtol=1e-5, atol=1e-7
|
||||
)
|
||||
|
||||
def test_roundtrip_non_contiguous_tensors(self):
|
||||
"""Test roundtrip with non-contiguous tensors from the trainer."""
|
||||
# Create non-contiguous tensors (simulating trainer outputs)
|
||||
# Transposed tensors are non-contiguous
|
||||
weight1 = torch.randn(20, 10, dtype=torch.float32).cuda().T
|
||||
# Sliced tensors with step are non-contiguous
|
||||
weight2 = torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]
|
||||
# Permuted tensors are non-contiguous
|
||||
weight3 = torch.randn(5, 10, 15, dtype=torch.bfloat16).cuda().permute(2, 0, 1)
|
||||
|
||||
params = [
|
||||
("layer1.weight", weight1),
|
||||
("layer2.weight", weight2),
|
||||
("layer3.weight", weight3),
|
||||
]
|
||||
|
||||
# Verify tensors are indeed non-contiguous
|
||||
for name, tensor in params:
|
||||
assert not tensor.is_contiguous(), f"{name} should be non-contiguous"
|
||||
|
||||
buffer_size = 500
|
||||
producer_group = MockCommunicationGroup()
|
||||
|
||||
packed_nccl_broadcast_producer(
|
||||
iterator=iter(params),
|
||||
group=producer_group,
|
||||
src=0,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
consumer_group = MockConsumerCommunicationGroup(
|
||||
producer_group.broadcasted_tensors
|
||||
)
|
||||
|
||||
state_dict_info = create_state_dict_info(params)
|
||||
unpacked_tensors = {}
|
||||
|
||||
def post_unpack_func(tensor_list):
|
||||
for name, tensor in tensor_list:
|
||||
unpacked_tensors[name] = tensor.clone()
|
||||
|
||||
packed_nccl_broadcast_consumer(
|
||||
iterator=iter(state_dict_info.items()),
|
||||
group=consumer_group,
|
||||
src=0,
|
||||
post_unpack_func=post_unpack_func,
|
||||
buffer_size_bytes=buffer_size,
|
||||
)
|
||||
|
||||
# Verify all non-contiguous params roundtrip correctly
|
||||
for name, original_tensor in params:
|
||||
assert name in unpacked_tensors
|
||||
unpacked = unpacked_tensors[name]
|
||||
assert unpacked.shape == original_tensor.shape
|
||||
assert unpacked.dtype == original_tensor.dtype
|
||||
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
|
||||
# --- Unit Tests: unpack_tensor ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestUnpackTensor:
|
||||
"""Test the shared unpack_tensor function."""
|
||||
|
||||
def test_unpack_produces_independent_copies(self):
|
||||
"""Verify unpacked tensors don't share memory with packed buffer."""
|
||||
original = torch.randn(10, dtype=torch.float32).cuda()
|
||||
packed = original.contiguous().view(torch.uint8).view(-1)
|
||||
|
||||
result = unpack_tensor(
|
||||
packed,
|
||||
names=["w"],
|
||||
shapes=[[10]],
|
||||
dtypes=[torch.float32],
|
||||
tensor_sizes=[packed.numel()],
|
||||
)
|
||||
|
||||
# Mutate the packed buffer
|
||||
packed.zero_()
|
||||
|
||||
# Unpacked tensor should be unaffected
|
||||
assert torch.allclose(result[0][1], original)
|
||||
|
||||
|
||||
# --- Unit Tests: pack_tensors ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackTensors:
|
||||
"""Test the shared pack_tensors function."""
|
||||
|
||||
def test_pack_basic(self):
|
||||
"""Test packing a few tensors into one buffer."""
|
||||
params = [
|
||||
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("w2", torch.randn(5, dtype=torch.float16).cuda()),
|
||||
]
|
||||
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
|
||||
assert chunk is not None
|
||||
assert len(chunk.names) == 2
|
||||
assert chunk.names == ["w1", "w2"]
|
||||
assert chunk.shapes == [[10, 20], [5]]
|
||||
assert chunk.dtypes == [torch.float32, torch.float16]
|
||||
assert chunk.packed_tensor.dtype == torch.uint8
|
||||
|
||||
def test_pack_respects_buffer_limit(self):
|
||||
"""Test that packing stops when buffer_size_bytes is exceeded."""
|
||||
params = [
|
||||
(f"w{i}", torch.randn(100, 100, dtype=torch.float32).cuda())
|
||||
for i in range(10)
|
||||
]
|
||||
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=50_000,
|
||||
)
|
||||
|
||||
assert chunk is not None
|
||||
assert len(chunk.names) < 10
|
||||
|
||||
def test_pack_empty_iterator(self):
|
||||
"""Test that an empty iterator returns None."""
|
||||
chunk = pack_tensors(
|
||||
iterator=iter([]),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
assert chunk is None
|
||||
|
||||
def test_pack_single_tensor_larger_than_buffer_warns(self):
|
||||
"""Test that a tensor exceeding buffer_size_bytes emits a warning."""
|
||||
big = torch.randn(1000, 1000, dtype=torch.float32).cuda()
|
||||
params = [("big", big)]
|
||||
|
||||
import warnings
|
||||
|
||||
with warnings.catch_warnings(record=True) as w:
|
||||
warnings.simplefilter("always")
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=100,
|
||||
)
|
||||
assert chunk is not None
|
||||
assert len(chunk.names) == 1
|
||||
assert any("exceeds buffer_size_bytes" in str(wi.message) for wi in w)
|
||||
|
||||
def test_pack_unpack_roundtrip(self):
|
||||
"""Test pack then unpack produces identical tensors."""
|
||||
params = [
|
||||
("a", torch.randn(8, 16, dtype=torch.float32).cuda()),
|
||||
("b", torch.randn(4, dtype=torch.float16).cuda()),
|
||||
("c", torch.randn(3, 5, 7, dtype=torch.bfloat16).cuda()),
|
||||
]
|
||||
|
||||
chunk = pack_tensors(
|
||||
iterator=iter(params),
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
|
||||
assert chunk is not None
|
||||
result = unpack_tensor(
|
||||
chunk.packed_tensor,
|
||||
chunk.names,
|
||||
chunk.shapes,
|
||||
chunk.dtypes,
|
||||
chunk.tensor_sizes,
|
||||
)
|
||||
|
||||
assert len(result) == len(params)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.shape == orig_tensor.shape
|
||||
assert res_tensor.dtype == orig_tensor.dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor, rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_pack_multiple_chunks(self):
|
||||
"""Test consuming an iterator across multiple pack_tensors calls."""
|
||||
params = [
|
||||
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
|
||||
]
|
||||
it = iter(params)
|
||||
|
||||
all_names = []
|
||||
chunks = []
|
||||
while True:
|
||||
chunk = pack_tensors(it, lambda x: x[1], buffer_size_bytes=12_000)
|
||||
if chunk is None:
|
||||
break
|
||||
chunks.append(chunk)
|
||||
all_names.extend(chunk.names)
|
||||
|
||||
assert len(chunks) > 1
|
||||
assert all_names == [f"w{i}" for i in range(6)]
|
||||
|
||||
|
||||
# --- Unit Tests: packed_ipc_producer ---
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedIpcProducer:
|
||||
"""Test the packed_ipc_producer generator."""
|
||||
|
||||
def test_producer_yields_chunks(self):
|
||||
"""Test that the producer yields PackedIpcChunk objects."""
|
||||
params = [
|
||||
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
|
||||
]
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid="test-uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=12_000,
|
||||
)
|
||||
)
|
||||
|
||||
assert len(chunks) > 1
|
||||
|
||||
def test_producer_ipc_handle_has_uuid(self):
|
||||
"""Test that each chunk's ipc_handle is keyed by the given UUID."""
|
||||
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid="my-gpu-uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
assert "my-gpu-uuid" in chunks[0].ipc_handle
|
||||
|
||||
def test_producer_dtype_names_are_strings(self):
|
||||
"""Test that dtype_names are string representations."""
|
||||
params = [
|
||||
("a", torch.randn(10, dtype=torch.float32).cuda()),
|
||||
("b", torch.randn(10, dtype=torch.float16).cuda()),
|
||||
]
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid="uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
assert chunks[0].dtype_names == ["float32", "float16"]
|
||||
|
||||
def test_producer_empty_iterator(self):
|
||||
"""Test producer with empty iterator yields nothing."""
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter([]),
|
||||
gpu_uuid="uuid",
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=1000,
|
||||
)
|
||||
)
|
||||
assert len(chunks) == 0
|
||||
|
||||
|
||||
# --- Integration Tests: IPC Producer-Consumer Roundtrip ---
|
||||
|
||||
|
||||
def _ipc_consumer_worker(cmd_q, ack_q, result_q, done_event, device_index):
|
||||
"""Worker that consumes chunks streamed one at a time from the parent.
|
||||
|
||||
CUDA IPC requires the consumer to be in a separate process from the
|
||||
producer. The producer reuses a single IPC buffer between chunks, so
|
||||
the parent must wait for our ack (sent after we copy the chunk to
|
||||
CPU) before advancing the producer.
|
||||
"""
|
||||
try:
|
||||
torch.accelerator.set_device_index(device_index)
|
||||
all_results = []
|
||||
while True:
|
||||
cd = cmd_q.get()
|
||||
if cd is None:
|
||||
break
|
||||
result = packed_ipc_consumer(
|
||||
ipc_handle=cd["ipc_handle"],
|
||||
names=cd["names"],
|
||||
shapes=cd["shapes"],
|
||||
dtype_names=cd["dtype_names"],
|
||||
tensor_sizes=cd["tensor_sizes"],
|
||||
device_index=device_index,
|
||||
)
|
||||
# .cpu() forces a GPU→CPU copy off the shared IPC buffer, so
|
||||
# the producer is free to overwrite it once we ack.
|
||||
all_results.extend([(name, tensor.cpu()) for name, tensor in result])
|
||||
del result
|
||||
ack_q.put("ack")
|
||||
result_q.put(("ok", all_results))
|
||||
except Exception as e:
|
||||
result_q.put(("error", str(e)))
|
||||
# Keep the process alive until the parent has finished reading from
|
||||
# the result queue — torch serializes CPU tensors via fd sharing,
|
||||
# which requires this process's resource-sharer server to be running.
|
||||
done_event.wait(timeout=60)
|
||||
|
||||
|
||||
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
|
||||
class TestPackedIpcRoundtrip:
|
||||
"""Test IPC producer-consumer roundtrip using real CUDA IPC.
|
||||
|
||||
These tests spawn a child process for the consumer because
|
||||
rebuild_cuda_tensor requires a separate process from the one that
|
||||
called reduce_tensor.
|
||||
"""
|
||||
|
||||
def _get_gpu_uuid(self) -> str:
|
||||
device_index = torch.cuda.current_device()
|
||||
props = torch.cuda.get_device_properties(device_index)
|
||||
return str(props.uuid)
|
||||
|
||||
def _run_roundtrip(self, chunk_iter, device_index, timeout=30):
|
||||
"""Stream chunks through a child consumer one at a time.
|
||||
|
||||
``packed_ipc_producer`` reuses a single IPC buffer for every
|
||||
chunk, so the producer must not be advanced until the consumer
|
||||
has finished reading the current chunk. We enforce that with an
|
||||
ack queue: the consumer puts ``"ack"`` after it has copied the
|
||||
chunk to CPU, and only then do we pull the next chunk from the
|
||||
generator.
|
||||
|
||||
Returns ``(num_chunks, results)``.
|
||||
"""
|
||||
import multiprocessing as mp
|
||||
|
||||
ctx = mp.get_context("spawn")
|
||||
cmd_q = ctx.Queue()
|
||||
ack_q = ctx.Queue()
|
||||
result_q = ctx.Queue()
|
||||
done_event = ctx.Event()
|
||||
proc = ctx.Process(
|
||||
target=_ipc_consumer_worker,
|
||||
args=(cmd_q, ack_q, result_q, done_event, device_index),
|
||||
)
|
||||
proc.start()
|
||||
|
||||
num_chunks = 0
|
||||
try:
|
||||
for chunk in chunk_iter:
|
||||
cmd_q.put(
|
||||
{
|
||||
"ipc_handle": chunk.ipc_handle,
|
||||
"names": chunk.names,
|
||||
"shapes": chunk.shapes,
|
||||
"dtype_names": chunk.dtype_names,
|
||||
"tensor_sizes": chunk.tensor_sizes,
|
||||
}
|
||||
)
|
||||
if ack_q.get(timeout=timeout) != "ack":
|
||||
raise RuntimeError("Consumer did not ack chunk")
|
||||
num_chunks += 1
|
||||
cmd_q.put(None)
|
||||
status, payload = result_q.get(timeout=timeout)
|
||||
finally:
|
||||
done_event.set()
|
||||
proc.join(timeout=10)
|
||||
if proc.is_alive():
|
||||
proc.kill()
|
||||
|
||||
if status == "error":
|
||||
raise RuntimeError(f"Consumer process failed: {payload}")
|
||||
# Reclaim IPC-shared memory now that the child has released it
|
||||
torch.cuda.ipc_collect()
|
||||
return num_chunks, payload
|
||||
|
||||
def test_roundtrip_basic(self):
|
||||
"""Test basic IPC producer -> consumer roundtrip."""
|
||||
params = [
|
||||
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
|
||||
("scale", torch.tensor(1.5, dtype=torch.float32).cuda()),
|
||||
("w2", torch.randn(5, dtype=torch.float16).cuda()),
|
||||
]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
num_chunks, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
assert num_chunks == 1
|
||||
assert len(result) == len(params)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.shape == orig_tensor.shape
|
||||
assert res_tensor.dtype == orig_tensor.dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_roundtrip_dtypes(self, dtype):
|
||||
"""Test IPC roundtrip with different dtypes."""
|
||||
params = create_mock_model_params(num_layers=2, dtype=dtype)
|
||||
params_cuda = [(n, t.cuda()) for n, t in params]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
_, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params_cuda),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
assert len(result) == len(params_cuda)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(
|
||||
params_cuda, result
|
||||
):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.dtype == dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_roundtrip_multiple_chunks(self):
|
||||
"""Test IPC roundtrip across multiple chunks."""
|
||||
params = [
|
||||
(f"layer{i}.weight", torch.randn(100, 100, dtype=torch.float32).cuda())
|
||||
for i in range(8)
|
||||
]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
num_chunks, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=50_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
assert num_chunks > 1
|
||||
assert len(result) == len(params)
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-5, atol=1e-7)
|
||||
|
||||
def test_roundtrip_non_contiguous(self):
|
||||
"""Test IPC roundtrip with non-contiguous tensors."""
|
||||
params = [
|
||||
("transposed", torch.randn(20, 10, dtype=torch.float32).cuda().T),
|
||||
("sliced", torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]),
|
||||
]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
device_index = torch.cuda.current_device()
|
||||
|
||||
for _, t in params:
|
||||
assert not t.is_contiguous()
|
||||
|
||||
_, result = self._run_roundtrip(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
),
|
||||
device_index,
|
||||
)
|
||||
|
||||
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
|
||||
assert orig_name == res_name
|
||||
assert res_tensor.shape == orig_tensor.shape
|
||||
assert res_tensor.dtype == orig_tensor.dtype
|
||||
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
|
||||
|
||||
def test_consumer_wrong_uuid_raises(self):
|
||||
"""Test that consumer raises ValueError for unknown GPU UUID."""
|
||||
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
|
||||
gpu_uuid = self._get_gpu_uuid()
|
||||
|
||||
chunks = list(
|
||||
packed_ipc_producer(
|
||||
iterator=iter(params),
|
||||
gpu_uuid=gpu_uuid,
|
||||
post_iter_func=lambda x: x[1],
|
||||
buffer_size_bytes=10_000_000,
|
||||
)
|
||||
)
|
||||
|
||||
c = chunks[0]
|
||||
fake_handle = {"fake-uuid-12345": c.ipc_handle[gpu_uuid]}
|
||||
|
||||
with pytest.raises(ValueError, match="IPC handle not found"):
|
||||
packed_ipc_consumer(
|
||||
ipc_handle=fake_handle,
|
||||
names=c.names,
|
||||
shapes=c.shapes,
|
||||
dtype_names=c.dtype_names,
|
||||
tensor_sizes=c.tensor_sizes,
|
||||
device_index=torch.cuda.current_device(),
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
|
||||
def test_pack_unpack_scalar(dtype):
|
||||
"""Preserve scalar shape, dtype and value alongside a matrix on CPU."""
|
||||
params = [
|
||||
("weight", torch.randn(2, 3, dtype=dtype)),
|
||||
("scale", torch.tensor(1.5, dtype=dtype)),
|
||||
]
|
||||
|
||||
chunk = pack_tensors(iter(params), lambda item: item[1], buffer_size_bytes=1024)
|
||||
assert chunk is not None
|
||||
|
||||
result = unpack_tensor(
|
||||
chunk.packed_tensor,
|
||||
chunk.names,
|
||||
chunk.shapes,
|
||||
chunk.dtypes,
|
||||
chunk.tensor_sizes,
|
||||
)
|
||||
|
||||
for (expected_name, expected), (name, actual) in zip(params, result, strict=True):
|
||||
assert name == expected_name
|
||||
assert actual.shape == expected.shape
|
||||
assert actual.dtype == expected.dtype
|
||||
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
|
||||
@@ -1,628 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the sharded-RDT trainer-side engine.
|
||||
|
||||
The trainer engine's own surface: init info and factory registration, the
|
||||
send_weights round trip against an in-process fake producer server, the
|
||||
ownership table it resolves and ships to the workers, and the producer's packed
|
||||
destination-view cache.
|
||||
|
||||
The plan/replay half (including `RdtRouter`, which is consumer-side) lives in
|
||||
test_sharded_rdt_plan.py; the producer server's protocol in
|
||||
test_sharded_rdt_producer.py.
|
||||
"""
|
||||
|
||||
import itertools
|
||||
from dataclasses import asdict
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from tests.distributed.test_weight_transfer import RecordingClient
|
||||
from vllm.distributed.weight_transfer import (
|
||||
ModuleSource,
|
||||
ParamMeta,
|
||||
WeightSource,
|
||||
WeightTransferTrainerFactory,
|
||||
)
|
||||
from vllm.distributed.weight_transfer.base import layerwise_groups
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
ShardedRDTTrainerInitInfo,
|
||||
ShardedRDTTrainerWeightTransferEngine,
|
||||
)
|
||||
|
||||
|
||||
class _ListSource(WeightSource):
|
||||
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
|
||||
sharded-RDT group/order logic can be tested without a real model."""
|
||||
|
||||
def __init__(self, pairs):
|
||||
self._pairs = list(pairs)
|
||||
|
||||
def metadata(self):
|
||||
return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs]
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._pairs)
|
||||
|
||||
|
||||
class _FakeProducerServer:
|
||||
"""In-process stand-in for the _RDTProducerServer Ray actor. Records the
|
||||
engine->server call sequence and, by default, frees each group as soon as
|
||||
it is published (simulating the consumers' free_group barrier) so the
|
||||
gather loop's credit gate never parks. Mirrors the real server's
|
||||
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
|
||||
to the ``begin_sync`` live total, publish_group returns nothing, and freed
|
||||
groups flow back only through wait_freed / end_sync (see
|
||||
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
|
||||
|
||||
def __init__(self, auto_free=True):
|
||||
self.order: list[str] = []
|
||||
self.published: list[int] = []
|
||||
self.live_count = 1
|
||||
self._inflight_groups: list[int] = []
|
||||
self.auto_free = auto_free
|
||||
self.free_counts: dict[int, int] = {}
|
||||
self._pending_freed: list[int] = []
|
||||
|
||||
# Alias for tests that inspect in-flight groups.
|
||||
@property
|
||||
def inflight(self):
|
||||
return self._inflight_groups
|
||||
|
||||
def begin_sync(self, live_count, live_consumer_ids=None):
|
||||
self.order.append("begin")
|
||||
self.live_count = max(1, int(live_count))
|
||||
self.live_ids = live_consumer_ids
|
||||
|
||||
def publish_group(self, group_idx, entries):
|
||||
self.order.append("publish")
|
||||
self.published.append(group_idx)
|
||||
self._inflight_groups.append(group_idx)
|
||||
if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count:
|
||||
self._inflight_groups.remove(group_idx)
|
||||
self._pending_freed.append(group_idx)
|
||||
|
||||
def free_group(self, group_idx):
|
||||
"""Consumer back-edge; may arrive before the group's publish."""
|
||||
self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1
|
||||
if (
|
||||
group_idx in self._inflight_groups
|
||||
and self.free_counts[group_idx] >= self.live_count
|
||||
):
|
||||
self._inflight_groups.remove(group_idx)
|
||||
self._pending_freed.append(group_idx)
|
||||
|
||||
def wait_freed(self):
|
||||
"""The engine's credit gate. The real server blocks here; the fake
|
||||
must already have a banked credit when the gate asks (auto_free, or a
|
||||
test's own free_group) — anything else is the deadlock the real
|
||||
watchdog would kill, so fail loudly. Not appended to ``order``: it is
|
||||
pacing, not a lifecycle milestone."""
|
||||
assert self._pending_freed, (
|
||||
"wait_freed with nothing freed: the gather loop would deadlock"
|
||||
)
|
||||
freed = self._pending_freed
|
||||
self._pending_freed = []
|
||||
return freed
|
||||
|
||||
def end_sync(self):
|
||||
self.order.append("end")
|
||||
freed = self._pending_freed
|
||||
self._pending_freed = []
|
||||
return freed
|
||||
|
||||
def set_gather_error(self, message):
|
||||
self.order.append("error")
|
||||
|
||||
|
||||
def _rdt_engine_with_fake_server(
|
||||
source, *, is_sender, client, server, monkeypatch, fleet_owned=None
|
||||
):
|
||||
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
|
||||
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
|
||||
group-major metadata, and route _rpc to the fake."""
|
||||
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
|
||||
|
||||
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
|
||||
monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",)))
|
||||
|
||||
init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1)
|
||||
engine = ShardedRDTTrainerWeightTransferEngine(
|
||||
client=client, source=source, is_sender=is_sender, init_info=init_info
|
||||
)
|
||||
engine._meta = list(source.metadata())
|
||||
names = [m.name for m in engine._meta]
|
||||
engine._groups = layerwise_groups(names)
|
||||
engine._server = server
|
||||
engine._rpc = lambda method, *args: getattr(server, method)(*args)
|
||||
# What trainer_init resolves from the source's held names + the fleet's
|
||||
# all-gather. ``fleet_owned`` stands in for that all-gather so a partial-
|
||||
# ownership rank can be tested without a real process group; the fleet must
|
||||
# cover every name or the transpose rejects it (nothing would serve the rest).
|
||||
if fleet_owned is None:
|
||||
engine._resolve_ownership(1, 0)
|
||||
else:
|
||||
# Stands in for the (metadata digest, held-name bitmask) all-gather:
|
||||
# each rank's group list becomes the mask its held names would set.
|
||||
names = [m.name for m in engine._meta]
|
||||
groups = engine._groups
|
||||
|
||||
def _fake_gather(_world, mine):
|
||||
out = []
|
||||
for owned in fleet_owned:
|
||||
held = {n for gi in owned for n in groups[gi]}
|
||||
mask = bytearray((len(names) + 7) // 8)
|
||||
for i, n in enumerate(names):
|
||||
if n in held:
|
||||
mask[i >> 3] |= 1 << (i & 7)
|
||||
out.append((mine[0], bytes(mask)))
|
||||
return out
|
||||
|
||||
monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather)
|
||||
engine._resolve_ownership(len(fleet_owned), 0)
|
||||
return engine
|
||||
|
||||
|
||||
def _rdt_source_two_layers():
|
||||
return _ListSource(
|
||||
[
|
||||
("embed.weight", torch.zeros(2)),
|
||||
("model.layers.0.w", torch.zeros(2)),
|
||||
("model.layers.1.w", torch.zeros(2)),
|
||||
("norm.weight", torch.zeros(2)),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
class TestShardedRDTTrainerInitInfo:
|
||||
def test_declares_backend(self):
|
||||
assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt"
|
||||
|
||||
def test_rank_is_keyword_only_and_drives_is_sender(self):
|
||||
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True
|
||||
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False
|
||||
with pytest.raises(TypeError):
|
||||
# rank is keyword-only.
|
||||
ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc]
|
||||
|
||||
def test_registered_in_trainer_factory(self):
|
||||
cls = WeightTransferTrainerFactory._registry["sharded_rdt"]()
|
||||
assert cls is ShardedRDTTrainerWeightTransferEngine
|
||||
|
||||
|
||||
def test_sharded_rdt_trainer_init_requires_source():
|
||||
with pytest.raises(ValueError, match="requires a WeightSource"):
|
||||
ShardedRDTTrainerWeightTransferEngine.trainer_init(
|
||||
ShardedRDTTrainerInitInfo(num_consumers=1, rank=0),
|
||||
client=RecordingClient(),
|
||||
source=None,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch):
|
||||
source = _rdt_source_two_layers()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
source,
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
worker_init = engine._build_worker_init_info(["srv_rk0"])
|
||||
# 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1.
|
||||
assert worker_init.names == [
|
||||
"embed.weight",
|
||||
"model.layers.0.w",
|
||||
"model.layers.1.w",
|
||||
"norm.weight",
|
||||
]
|
||||
assert worker_init.group_lens == [1, 1, 1, 1]
|
||||
assert worker_init.trainer_actor_names == ["srv_rk0"]
|
||||
assert worker_init.produce_method_name == "rdt_produce_weights_batched"
|
||||
assert sum(worker_init.group_lens) == len(worker_init.names)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch):
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
client = RecordingClient()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=True,
|
||||
client=client,
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights()
|
||||
|
||||
assert client.order == ["start", "update", "finish"]
|
||||
# begin, one publish per group (4), end.
|
||||
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
|
||||
assert len(server.published) == 4
|
||||
# every group freed -> no engine-held refs remain.
|
||||
assert engine._inflight == {}
|
||||
|
||||
|
||||
def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch):
|
||||
# Source whose iteration order disagrees with its metadata order.
|
||||
class _BadSource(_ListSource):
|
||||
def __iter__(self):
|
||||
reordered = list(self._pairs)
|
||||
reordered[0], reordered[1] = reordered[1], reordered[0]
|
||||
return iter(reordered)
|
||||
|
||||
server = _FakeProducerServer()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_BadSource(_rdt_source_two_layers()._pairs),
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
engine.send_weights()
|
||||
assert "error" in server.order # gather error propagated to the server
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_non_sender_skips_client(monkeypatch):
|
||||
class _RaisingClient(RecordingClient):
|
||||
def start_weight_update(self):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
def update_weights(self, update_info):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
def finish_weight_update(self, weight_version: str | None = None):
|
||||
raise AssertionError("non-sender must not touch the client")
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
client = _RaisingClient()
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=False,
|
||||
client=client,
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights() # gathers only; must not raise
|
||||
assert client.order == []
|
||||
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
|
||||
class _FailingUpdateClient(RecordingClient):
|
||||
def update_weights(self, update_info):
|
||||
self.order.append("update")
|
||||
raise RuntimeError("inference side rejected update")
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=True,
|
||||
client=_FailingUpdateClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="inference side rejected update"):
|
||||
engine.send_weights()
|
||||
|
||||
|
||||
class _OwnedSource(_ListSource):
|
||||
"""A source holding only some groups' names, like a pipeline-parallel rank.
|
||||
Takes group indices for convenience and declares the names inside them."""
|
||||
|
||||
def __init__(self, pairs, owned_group_idx):
|
||||
super().__init__(pairs)
|
||||
self._owned = list(owned_group_idx)
|
||||
groups = layerwise_groups([n for n, _ in pairs])
|
||||
self._owned_names = [n for gi in self._owned for n in groups[gi]]
|
||||
|
||||
def held_names(self):
|
||||
return list(self._owned_names)
|
||||
|
||||
def __iter__(self):
|
||||
by_name = dict(self._pairs)
|
||||
return iter([(n, by_name[n]) for n in self._owned_names])
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch):
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest
|
||||
)
|
||||
engine.send_weights()
|
||||
|
||||
assert server.published == [1, 2]
|
||||
assert server.order == ["begin", "publish", "publish", "end"]
|
||||
assert engine._inflight == {}
|
||||
# embed + norm on rank 1, the two layers here: two distinct owner sets,
|
||||
# numbered by first appearance in metadata order.
|
||||
assert engine._owner_sets == [[1], [0]]
|
||||
assert engine._name_owner_class == [0, 1, 1, 0]
|
||||
|
||||
|
||||
def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
|
||||
class _MisorderedOwned(_OwnedSource):
|
||||
def __iter__(self):
|
||||
return iter(list(super().__iter__())[::-1])
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]],
|
||||
)
|
||||
with pytest.raises(RuntimeError, match="iteration order must match"):
|
||||
engine.send_weights()
|
||||
assert "error" in server.order
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(),
|
||||
reason="the gather loop's CUDA-IPC export needs a device",
|
||||
)
|
||||
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
|
||||
"""The free barrier's target is one integer per sync: the live consumer
|
||||
count, defaulting to the whole provisioned fleet."""
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_rdt_source_two_layers(),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
engine.send_weights()
|
||||
assert server.live_count == 1 # num_consumers=1 in the harness
|
||||
engine.send_weights(live_consumer_ids=[0])
|
||||
assert server.live_count == 1
|
||||
|
||||
|
||||
class TestLiveCountPlumbing:
|
||||
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
|
||||
provisioned geometry is frozen; a degraded sync only lowers the target and
|
||||
narrows each slot-sharing group's rendezvous."""
|
||||
|
||||
@staticmethod
|
||||
def _engine(num_consumers, world=2, rank=0):
|
||||
engine = ShardedRDTTrainerWeightTransferEngine.__new__(
|
||||
ShardedRDTTrainerWeightTransferEngine
|
||||
)
|
||||
engine._init_info = ShardedRDTTrainerInitInfo(
|
||||
rank=rank, num_consumers=num_consumers
|
||||
)
|
||||
engine.source = object() # send_weights asserts a source is present
|
||||
received: list = []
|
||||
engine._send_weights_inner = lambda count, ids: received.append((count, ids))
|
||||
return engine, received
|
||||
|
||||
def test_none_counts_the_whole_provisioned_fleet(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(None)
|
||||
assert got == [(8, list(range(8)))]
|
||||
|
||||
def test_a_live_set_counts_its_distinct_members(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([0, 1, 4, 5, 5])
|
||||
assert got == [(4, [0, 1, 4, 5])]
|
||||
|
||||
def test_a_full_live_set_matches_the_provisioned_count(self):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(list(range(8)))
|
||||
engine.send_weights(None)
|
||||
assert got == [(8, list(range(8)))] * 2
|
||||
|
||||
def test_which_consumers_died_does_not_matter_only_how_many(self):
|
||||
"""The whole point of the barrier: no routed per-producer targets, so
|
||||
the identity of the dead consumer is irrelevant to the producers."""
|
||||
counts = []
|
||||
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights(live)
|
||||
counts += [c for c, _ids in got]
|
||||
assert counts == [4, 4, 4]
|
||||
|
||||
def test_the_live_ids_travel_with_the_count(self):
|
||||
"""The count sizes the free barrier, the ids size the slot-sharing
|
||||
rendezvous, so a producer that shares slots can tell WHICH consumers it
|
||||
is still waiting for. They must describe the same set."""
|
||||
engine, got = self._engine(num_consumers=8)
|
||||
engine.send_weights([6, 0, 2, 2])
|
||||
((count, ids),) = got
|
||||
assert ids == [0, 2, 6] and count == len(ids)
|
||||
|
||||
|
||||
def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch):
|
||||
import json
|
||||
|
||||
server = _FakeProducerServer(auto_free=True)
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
is_sender=True,
|
||||
client=RecordingClient(),
|
||||
server=server,
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[0, 1], [2, 3]],
|
||||
)
|
||||
worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"])
|
||||
assert worker_init.owner_sets == [[0], [1]]
|
||||
assert worker_init.name_owner_class == [0, 0, 1, 1]
|
||||
assert len(worker_init.name_owner_class) == len(worker_init.names)
|
||||
# The payload crosses the control plane as JSON: nested lists must survive.
|
||||
round_tripped = json.loads(json.dumps(asdict(worker_init)))
|
||||
assert round_tripped["owner_sets"] == [[0], [1]]
|
||||
assert round_tripped["name_owner_class"] == [0, 0, 1, 1]
|
||||
|
||||
|
||||
def test_weight_source_holds_everything_by_default():
|
||||
"""The contract's default: a source that says nothing holds the whole model."""
|
||||
src = _rdt_source_two_layers()
|
||||
assert src.held_names() is None
|
||||
assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch):
|
||||
class _BadHeld(_OwnedSource):
|
||||
def held_names(self):
|
||||
return ["embed.weight", "not.a.real.name"]
|
||||
|
||||
with pytest.raises(ValueError, match="not"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_BadHeld(_rdt_source_two_layers()._pairs, [0]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
|
||||
class _HoldsNothing(_ListSource):
|
||||
def held_names(self):
|
||||
return []
|
||||
|
||||
with pytest.raises(ValueError, match="empty"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_HoldsNothing(_rdt_source_two_layers()._pairs),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
|
||||
"""Every name must be held somewhere or it can never be served — caught
|
||||
when the holdings are transposed, naming the orphan."""
|
||||
with pytest.raises(ValueError, match="no trainer rank holds"):
|
||||
_rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(),
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[0, 1], [2]], # nobody holds group 3
|
||||
)
|
||||
|
||||
|
||||
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
|
||||
"""Only the sender's metadata reaches the consumers, so a rank describing
|
||||
just its own share must fail loudly rather than silently drop the rest."""
|
||||
engine = _rdt_engine_with_fake_server(
|
||||
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
|
||||
is_sender=False,
|
||||
client=RecordingClient(),
|
||||
server=_FakeProducerServer(auto_free=True),
|
||||
monkeypatch=monkeypatch,
|
||||
fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds
|
||||
)
|
||||
# Now rank 1 reports a DIFFERENT metadata digest for the same model.
|
||||
monkeypatch.setattr(
|
||||
engine,
|
||||
"_all_gather_owned",
|
||||
lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])],
|
||||
)
|
||||
with pytest.raises(ValueError, match="disagrees across trainer ranks"):
|
||||
engine._resolve_ownership(2, 0)
|
||||
|
||||
|
||||
def _serve_ring_server(src_name, src):
|
||||
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
|
||||
pull needs no Ray and no NIXL registration. Returns (server, serve) where
|
||||
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
|
||||
what puts the destination-view cache, and only it, under test."""
|
||||
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
|
||||
_RDTProducerServer,
|
||||
)
|
||||
|
||||
srv = _RDTProducerServer(
|
||||
num_rdt_buffers=2,
|
||||
buffer_presize_gb=0.0,
|
||||
gather_lookahead=2,
|
||||
)
|
||||
srv._cache[src_name] = src
|
||||
srv._serve_rings[0] = [
|
||||
torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2)
|
||||
]
|
||||
|
||||
# The slot is ``seq % nring``, so stepping seq by nring keeps every pull in
|
||||
# slot 0 while still making each one its own generation -- reusing a seq
|
||||
# would rendezvous with the finished pack and return it unchanged.
|
||||
seqs = itertools.count(0, srv._nring)
|
||||
|
||||
def serve(chain):
|
||||
return srv.rdt_produce_weights_batched(
|
||||
[(src_name, chain)], consumer_id=0, seq=next(seqs)
|
||||
)[0]
|
||||
|
||||
return srv, serve
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
|
||||
)
|
||||
def test_serve_does_not_reuse_packed_views_of_another_shape():
|
||||
"""Two requests can share a name yet pack different slices of it.
|
||||
|
||||
The producer caches the destination views it carves into a serve ring slot.
|
||||
Keyed by name alone, the second request is packed through the first's views.
|
||||
Reachable when one name's copies split across owner-class chunks.
|
||||
"""
|
||||
name = "model.layers.0.w"
|
||||
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
|
||||
_srv, serve = _serve_ring_server(name, src)
|
||||
|
||||
serve((("narrow", (0, 0, 2), ()),)) # 2 rows
|
||||
wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot
|
||||
blob = serve((("narrow", (0, 0, 6), ()),))
|
||||
|
||||
got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8)
|
||||
assert torch.equal(got, wide)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
|
||||
)
|
||||
def test_serve_does_not_reuse_packed_views_of_another_dtype():
|
||||
"""The SILENT case of the same cache: two requests whose slices have the same
|
||||
name and the same shape but different dtypes pack at identical offsets, so
|
||||
reusing the stale views raises nothing — ``copy_`` just casts, and the blob
|
||||
carries the wrong bytes with no check downstream.
|
||||
|
||||
``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under
|
||||
``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte
|
||||
length identical, which is exactly what makes the stale view fit.
|
||||
"""
|
||||
name = "model.layers.0.w"
|
||||
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
|
||||
_srv, serve = _serve_ring_server(name, src)
|
||||
|
||||
serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16
|
||||
blob = serve(()) # same name and shape, bf16
|
||||
|
||||
got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8)
|
||||
assert torch.equal(got, src), "packed through a cached view of the wrong dtype"
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,465 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous.
|
||||
|
||||
The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX
|
||||
trainer) initialize NCCL weight transfer from a unique id minted out of band.
|
||||
|
||||
* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL
|
||||
init infos.
|
||||
* Two-GPU integration tests driving the real worker engine against a peer: one
|
||||
via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer
|
||||
(cupy buffer + raw ``ncclBroadcast``).
|
||||
|
||||
The shared Ray helpers come from the sibling module (unchanged); the worker task
|
||||
is local so the sibling's TCP test and its worker stay untouched.
|
||||
"""
|
||||
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import ray
|
||||
import torch
|
||||
|
||||
from tests.distributed.test_weight_transfer import (
|
||||
_init_ray_for_weight_transfer,
|
||||
_set_ray_assigned_device,
|
||||
create_mock_vllm_config,
|
||||
)
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerInitInfo,
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
)
|
||||
|
||||
# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId.
|
||||
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
|
||||
|
||||
|
||||
class TestNCCLRendezvousValidation:
|
||||
"""Test the xor-of-rendezvous-modes validation on the NCCL init infos."""
|
||||
|
||||
def test_worker_init_info_is_keyword_only(self):
|
||||
# kw_only so a stale positional call fails loudly rather than silently
|
||||
# swapping fields once the rendezvous fields became optional.
|
||||
with pytest.raises(TypeError):
|
||||
NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg]
|
||||
|
||||
def test_trainer_init_info_accepts_positional_fields(self):
|
||||
# NCCLTrainerInitInfo keeps positional subclass fields; only `rank`
|
||||
# (from TrainerInitInfo) is keyword-only.
|
||||
trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0)
|
||||
trainer_keyword = NCCLTrainerInitInfo(
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
world_size=3,
|
||||
rank=0,
|
||||
)
|
||||
assert trainer_positional == trainer_keyword
|
||||
|
||||
def test_tcp_mode_valid(self):
|
||||
info = NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
)
|
||||
assert info.master_address == "127.0.0.1"
|
||||
assert info.nccl_unique_id_b64 is None
|
||||
|
||||
def test_uid_mode_valid(self):
|
||||
info = NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64=VALID_UID_B64,
|
||||
)
|
||||
assert info.nccl_unique_id_b64 == VALID_UID_B64
|
||||
assert info.master_address is None
|
||||
assert info.master_port is None
|
||||
assert VALID_UID_B64 not in repr(info)
|
||||
|
||||
def test_both_modes_raises(self):
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
master_port=12345,
|
||||
nccl_unique_id_b64=VALID_UID_B64,
|
||||
)
|
||||
|
||||
def test_neither_mode_raises(self):
|
||||
with pytest.raises(ValueError, match="need nccl_unique_id_b64"):
|
||||
NCCLWeightTransferInitInfo(rank_offset=1, world_size=3)
|
||||
|
||||
def test_half_specified_tcp_raises(self):
|
||||
with pytest.raises(ValueError, match="master_port"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
master_address="127.0.0.1",
|
||||
)
|
||||
|
||||
def test_bad_base64_uid_raises(self):
|
||||
with pytest.raises(ValueError, match="not valid standard base64"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64="!!!!",
|
||||
)
|
||||
|
||||
def test_wrong_length_uid_raises(self):
|
||||
short_uid = base64.b64encode(b"\x00" * 64).decode()
|
||||
with pytest.raises(ValueError, match="128"):
|
||||
NCCLWeightTransferInitInfo(
|
||||
rank_offset=1,
|
||||
world_size=3,
|
||||
nccl_unique_id_b64=short_uid,
|
||||
)
|
||||
|
||||
def test_parse_init_info_uid_valid(self):
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
init_info = engine.parse_init_info(
|
||||
{
|
||||
"nccl_unique_id_b64": VALID_UID_B64,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
assert isinstance(init_info, NCCLWeightTransferInitInfo)
|
||||
assert init_info.nccl_unique_id_b64 == VALID_UID_B64
|
||||
assert init_info.master_address is None
|
||||
|
||||
def test_parse_init_info_both_modes_raises(self):
|
||||
engine = NCCLWeightTransferEngine(
|
||||
WeightTransferConfig(backend="nccl"),
|
||||
create_mock_vllm_config(),
|
||||
torch.device("cuda"),
|
||||
MagicMock(spec=torch.nn.Module),
|
||||
)
|
||||
with pytest.raises(ValueError, match="not both"):
|
||||
engine.parse_init_info(
|
||||
{
|
||||
"master_address": "127.0.0.1",
|
||||
"master_port": 12345,
|
||||
"nccl_unique_id_b64": VALID_UID_B64,
|
||||
"rank_offset": 1,
|
||||
"world_size": 3,
|
||||
}
|
||||
)
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("unique_id_bytes", "rank", "world_size", "match"),
|
||||
[
|
||||
(b"\x00" * 64, 0, 2, "128-byte"),
|
||||
(b"\x00" * 128, -1, 2, "out of range"),
|
||||
(b"\x00" * 128, 2, 2, "out of range"),
|
||||
],
|
||||
)
|
||||
def test_from_unique_id_bytes_validation(
|
||||
self, unique_id_bytes, rank, world_size, match
|
||||
):
|
||||
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
|
||||
|
||||
with pytest.raises(ValueError, match=match):
|
||||
PyNcclCommunicator.from_unique_id_bytes(
|
||||
unique_id_bytes,
|
||||
rank=rank,
|
||||
world_size=world_size,
|
||||
device=0,
|
||||
)
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def inference_receive_tensor(
|
||||
init_info_dict: dict,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Worker that joins via the given init dict and records the tensor it gets.
|
||||
|
||||
Local to this module (rather than the sibling's TCP-only worker) so it can
|
||||
take a UID init payload -- the same dict boundary an external trainer POSTs.
|
||||
"""
|
||||
import contextlib
|
||||
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
|
||||
from vllm.config.parallel import ParallelConfig
|
||||
from vllm.config.weight_transfer import WeightTransferConfig
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLWeightTransferEngine,
|
||||
NCCLWeightTransferInitInfo,
|
||||
NCCLWeightTransferUpdateInfo,
|
||||
)
|
||||
|
||||
class Recorder(torch.nn.Module):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.received = []
|
||||
|
||||
def load_weights(self, weights):
|
||||
for name, tensor in weights:
|
||||
self.received.append((name, tensor.clone()))
|
||||
|
||||
config = WeightTransferConfig(backend="nccl")
|
||||
vllm_config = MagicMock()
|
||||
parallel_config = MagicMock(spec=ParallelConfig)
|
||||
parallel_config.rank = 0
|
||||
parallel_config.world_size = 1
|
||||
parallel_config.data_parallel_rank = 0
|
||||
parallel_config.data_parallel_index = 0
|
||||
vllm_config.parallel_config = parallel_config
|
||||
vllm_config.model_config = MagicMock()
|
||||
|
||||
recorder = Recorder()
|
||||
engine = NCCLWeightTransferEngine(
|
||||
config, vllm_config, torch.device("cuda"), recorder
|
||||
)
|
||||
# Transport-only test: bypass the set_current_vllm_config context that
|
||||
# receive_weights enters, since vllm_config here is a mock.
|
||||
import vllm.config as _vllm_config_mod
|
||||
|
||||
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
|
||||
|
||||
try:
|
||||
engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict))
|
||||
engine.receive_weights(
|
||||
NCCLWeightTransferUpdateInfo(
|
||||
names=["test.weight"],
|
||||
dtype_names=[tensor_dtype],
|
||||
shapes=[tensor_shape],
|
||||
)
|
||||
)
|
||||
torch.accelerator.synchronize()
|
||||
|
||||
received_shape = None
|
||||
received_sum = None
|
||||
if len(recorder.received) == 1:
|
||||
_name, tensor = recorder.received[0]
|
||||
received_shape = list(tensor.shape)
|
||||
received_sum = tensor.sum().item()
|
||||
|
||||
return {
|
||||
"success": received_shape == tensor_shape
|
||||
and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01,
|
||||
"received_shape": received_shape,
|
||||
"received_sum": received_sum,
|
||||
}
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor_uid(
|
||||
uid_b64: str,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Drive rank 0 via vLLM's own communicator from a pre-shared unique id.
|
||||
|
||||
A real torch-free trainer would mint the id and drive rank 0 itself; here we
|
||||
reuse the real send protocol by injecting the communicator into a sender
|
||||
engine, exercising the worker-side UID path end to end.
|
||||
"""
|
||||
import pybase64 as base64
|
||||
import torch
|
||||
|
||||
_set_ray_assigned_device()
|
||||
|
||||
from vllm.distributed.weight_transfer import ModuleSource
|
||||
from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group
|
||||
from vllm.distributed.weight_transfer.nccl_engine import (
|
||||
NCCLTrainerWeightTransferEngine,
|
||||
)
|
||||
|
||||
class NoopClient:
|
||||
def init_weight_transfer_engine(self, init_info):
|
||||
pass
|
||||
|
||||
def start_weight_update(self):
|
||||
pass
|
||||
|
||||
def update_weights(self, update_info):
|
||||
pass
|
||||
|
||||
def finish_weight_update(self, weight_version=None):
|
||||
pass
|
||||
|
||||
dtype = getattr(torch, tensor_dtype)
|
||||
model = torch.nn.Module()
|
||||
test_module = torch.nn.Module()
|
||||
test_module.register_parameter(
|
||||
"weight",
|
||||
torch.nn.Parameter(
|
||||
torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False
|
||||
),
|
||||
)
|
||||
model.add_module("test", test_module)
|
||||
|
||||
engine = NCCLTrainerWeightTransferEngine(
|
||||
client=NoopClient(),
|
||||
source=ModuleSource(model),
|
||||
is_sender=True,
|
||||
packed=False,
|
||||
)
|
||||
engine.model_update_group = uid_init_process_group(
|
||||
base64.b64decode(uid_b64),
|
||||
rank=0,
|
||||
world_size=world_size,
|
||||
device=torch.accelerator.current_device_index(),
|
||||
)
|
||||
try:
|
||||
engine.send_weights()
|
||||
torch.accelerator.synchronize()
|
||||
return {"success": True}
|
||||
finally:
|
||||
engine.shutdown()
|
||||
|
||||
|
||||
@ray.remote(num_gpus=1)
|
||||
def trainer_broadcast_tensor_torch_free(
|
||||
uid_b64: str,
|
||||
world_size: int,
|
||||
tensor_shape: list[int],
|
||||
tensor_dtype: str,
|
||||
) -> dict:
|
||||
"""Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in).
|
||||
|
||||
Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it
|
||||
exercises interop with a foreign NCCL peer rather than vLLM against itself.
|
||||
``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right
|
||||
after ``ncclCommInitRank``; that is a collective, so this peer must issue the
|
||||
matching all_reduce before the broadcast or both ranks deadlock.
|
||||
"""
|
||||
import cupy as cp
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
|
||||
device = _set_ray_assigned_device()
|
||||
cp.cuda.Device(device.index).use()
|
||||
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import (
|
||||
NCCLLibrary,
|
||||
buffer_type,
|
||||
cudaStream_t,
|
||||
ncclDataTypeEnum,
|
||||
ncclRedOpTypeEnum,
|
||||
)
|
||||
|
||||
assert tensor_dtype == "float32", "harness only wires float32"
|
||||
# Fill on the host and copy up (a plain H2D memcpy) so we neither use torch
|
||||
# nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets
|
||||
# the worker verify by sum == numel.
|
||||
sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32))
|
||||
warmup = cp.asarray(np.zeros(1, dtype=np.float32))
|
||||
numel = int(sendbuff.size)
|
||||
# The copies above run on the default stream; drain them before the NCCL
|
||||
# collectives read the buffers on our own stream.
|
||||
cp.cuda.Device(device.index).synchronize()
|
||||
stream = cp.cuda.Stream()
|
||||
|
||||
nccl = NCCLLibrary()
|
||||
comm = nccl.ncclCommInitRank(
|
||||
world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0
|
||||
)
|
||||
try:
|
||||
# Mirror the worker's init-time warm-up all_reduce (rank 0 must join it).
|
||||
nccl.ncclAllReduce(
|
||||
buffer_type(warmup.data.ptr),
|
||||
buffer_type(warmup.data.ptr),
|
||||
1,
|
||||
ncclDataTypeEnum.ncclFloat32,
|
||||
ncclRedOpTypeEnum.ncclSum,
|
||||
comm,
|
||||
cudaStream_t(stream.ptr),
|
||||
)
|
||||
nccl.ncclBroadcast(
|
||||
buffer_type(sendbuff.data.ptr),
|
||||
buffer_type(sendbuff.data.ptr),
|
||||
numel,
|
||||
ncclDataTypeEnum.ncclFloat32,
|
||||
0, # root
|
||||
comm,
|
||||
cudaStream_t(stream.ptr),
|
||||
)
|
||||
stream.synchronize()
|
||||
return {"success": True}
|
||||
finally:
|
||||
# Abort (not the collective destroy) for an uncoordinated teardown.
|
||||
nccl.ncclCommAbort(comm)
|
||||
|
||||
|
||||
def _run_uid_transfer(trainer_task) -> None:
|
||||
"""Mint a unique id, run the worker and the given trainer peer concurrently,
|
||||
and assert the tensor round-trips. Both ranks must enter init together --
|
||||
there is no store barrier on the UID path."""
|
||||
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
|
||||
|
||||
_init_ray_for_weight_transfer()
|
||||
|
||||
nccl = NCCLLibrary()
|
||||
uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode()
|
||||
|
||||
world_size = 2 # 1 trainer + 1 inference worker
|
||||
tensor_shape = [100, 100]
|
||||
tensor_dtype = "float32"
|
||||
|
||||
inference_future = inference_receive_tensor.remote(
|
||||
{
|
||||
"rank_offset": 1,
|
||||
"world_size": world_size,
|
||||
"nccl_unique_id_b64": uid_b64,
|
||||
"packed": False,
|
||||
},
|
||||
tensor_shape,
|
||||
tensor_dtype,
|
||||
)
|
||||
trainer_future = trainer_task.remote(
|
||||
uid_b64, world_size, tensor_shape, tensor_dtype
|
||||
)
|
||||
|
||||
try:
|
||||
trainer_result, result = ray.get(
|
||||
[trainer_future, inference_future], timeout=120
|
||||
)
|
||||
finally:
|
||||
ray.shutdown()
|
||||
|
||||
assert trainer_result["success"], "Trainer should complete successfully"
|
||||
assert result["success"], (
|
||||
f"Weight transfer failed. "
|
||||
f"Received shape: {result['received_shape']}, "
|
||||
f"Received sum: {result['received_sum']}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_between_processes_uid():
|
||||
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
|
||||
ranks on vLLM's ``PyNcclCommunicator``."""
|
||||
_run_uid_transfer(trainer_broadcast_tensor_uid)
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
torch.accelerator.device_count() < 2,
|
||||
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
|
||||
)
|
||||
def test_nccl_weight_transfer_torch_free_trainer():
|
||||
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
|
||||
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
|
||||
pytest.importorskip("cupy")
|
||||
_run_uid_transfer(trainer_broadcast_tensor_torch_free)
|
||||
Reference in New Issue
Block a user