[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
-775
View File
@@ -1,775 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for packed tensor broadcasting functionality.
Unit tests for packed_nccl_broadcast_producer and packed_nccl_broadcast_consumer.
These utilities enable efficient batched tensor transfer over NCCL.
"""
import pytest
import torch
from vllm.distributed.weight_transfer.packed_tensor import (
pack_tensors,
packed_ipc_consumer,
packed_ipc_producer,
packed_nccl_broadcast_consumer,
packed_nccl_broadcast_producer,
unpack_tensor,
)
class MockCommunicationGroup:
"""Mock communication group for testing producer broadcast operations."""
def __init__(self):
self.broadcasted_tensors: list[torch.Tensor] = []
self.broadcast_count = 0
self.device = torch.device("cuda:0")
def broadcast(self, tensor, src):
"""Mock broadcast that stores the tensor for later verification."""
self.broadcasted_tensors.append(tensor.clone())
self.broadcast_count += 1
class MockConsumerCommunicationGroup:
"""Mock communication group for consumer that returns pre-stored tensors."""
def __init__(self, tensors_to_return: list[torch.Tensor]):
self.tensors_to_return = tensors_to_return
self.current_index = 0
self.device = torch.device("cuda:0")
def broadcast(self, tensor, src):
"""Mock broadcast that fills the tensor with pre-stored data."""
if self.current_index < len(self.tensors_to_return):
tensor.copy_(self.tensors_to_return[self.current_index])
self.current_index += 1
def create_mock_model_params(
num_layers: int = 3,
dtype: torch.dtype = torch.float32,
) -> list[tuple[str, torch.Tensor]]:
"""Create mock model parameters for testing."""
params = []
for i in range(num_layers):
params.append((f"layer{i}.weight", torch.randn(10, 20, dtype=dtype)))
params.append((f"layer{i}.bias", torch.randn(10, dtype=dtype)))
return params
def create_state_dict_info(
params: list[tuple[str, torch.Tensor]],
) -> dict[str, tuple[tuple[int, ...], torch.dtype]]:
"""Create state dict info (name -> (shape, dtype)) from params."""
return {name: (tuple(tensor.shape), tensor.dtype) for name, tensor in params}
# --- Unit Tests: packed_nccl_broadcast_producer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedBroadcastProducer:
"""Test packed_nccl_broadcast_producer function."""
def test_producer_empty_iterator(self):
"""Test producer handles empty iterator gracefully."""
mock_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter([]),
group=mock_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
# No broadcasts for empty iterator
assert mock_group.broadcast_count == 0
# --- Integration Tests: Producer-Consumer Roundtrip ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedBroadcastRoundtrip:
"""Test producer-consumer roundtrip behavior."""
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_roundtrip_different_dtypes(self, dtype):
"""Test roundtrip with different data types."""
params = create_mock_model_params(num_layers=2, dtype=dtype)
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
buffer_size = 1000
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params_cuda)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify roundtrip preserves data
for name, original_tensor in params_cuda:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.dtype == dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
def test_roundtrip_mixed_dtypes(self):
"""Test roundtrip with mixed data types."""
# Create params with mixed dtypes
params = [
("layer1.weight", torch.randn(10, 20, dtype=torch.float32).cuda()),
("layer1.bias", torch.randn(10, dtype=torch.float16).cuda()),
("layer2.weight", torch.randn(20, 30, dtype=torch.bfloat16).cuda()),
]
buffer_size = 500
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify all params roundtrip correctly with correct dtypes
for name, original_tensor in params:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.shape == original_tensor.shape
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
@pytest.mark.parametrize("target_size", [100, 100000])
def test_roundtrip_different_batch_sizes(self, target_size):
"""Test roundtrip with different target batch sizes."""
params = create_mock_model_params(num_layers=5)
params_cuda = [(name, tensor.cuda()) for name, tensor in params]
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params_cuda),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=target_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params_cuda)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=target_size,
)
# Verify all params roundtrip correctly
assert len(unpacked_tensors) == len(params)
for name, original_tensor in params_cuda:
assert name in unpacked_tensors
assert torch.allclose(
unpacked_tensors[name], original_tensor, rtol=1e-5, atol=1e-7
)
def test_roundtrip_non_contiguous_tensors(self):
"""Test roundtrip with non-contiguous tensors from the trainer."""
# Create non-contiguous tensors (simulating trainer outputs)
# Transposed tensors are non-contiguous
weight1 = torch.randn(20, 10, dtype=torch.float32).cuda().T
# Sliced tensors with step are non-contiguous
weight2 = torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]
# Permuted tensors are non-contiguous
weight3 = torch.randn(5, 10, 15, dtype=torch.bfloat16).cuda().permute(2, 0, 1)
params = [
("layer1.weight", weight1),
("layer2.weight", weight2),
("layer3.weight", weight3),
]
# Verify tensors are indeed non-contiguous
for name, tensor in params:
assert not tensor.is_contiguous(), f"{name} should be non-contiguous"
buffer_size = 500
producer_group = MockCommunicationGroup()
packed_nccl_broadcast_producer(
iterator=iter(params),
group=producer_group,
src=0,
post_iter_func=lambda x: x[1],
buffer_size_bytes=buffer_size,
)
consumer_group = MockConsumerCommunicationGroup(
producer_group.broadcasted_tensors
)
state_dict_info = create_state_dict_info(params)
unpacked_tensors = {}
def post_unpack_func(tensor_list):
for name, tensor in tensor_list:
unpacked_tensors[name] = tensor.clone()
packed_nccl_broadcast_consumer(
iterator=iter(state_dict_info.items()),
group=consumer_group,
src=0,
post_unpack_func=post_unpack_func,
buffer_size_bytes=buffer_size,
)
# Verify all non-contiguous params roundtrip correctly
for name, original_tensor in params:
assert name in unpacked_tensors
unpacked = unpacked_tensors[name]
assert unpacked.shape == original_tensor.shape
assert unpacked.dtype == original_tensor.dtype
assert torch.allclose(unpacked, original_tensor, rtol=1e-4, atol=1e-6)
# --- Unit Tests: unpack_tensor ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestUnpackTensor:
"""Test the shared unpack_tensor function."""
def test_unpack_produces_independent_copies(self):
"""Verify unpacked tensors don't share memory with packed buffer."""
original = torch.randn(10, dtype=torch.float32).cuda()
packed = original.contiguous().view(torch.uint8).view(-1)
result = unpack_tensor(
packed,
names=["w"],
shapes=[[10]],
dtypes=[torch.float32],
tensor_sizes=[packed.numel()],
)
# Mutate the packed buffer
packed.zero_()
# Unpacked tensor should be unaffected
assert torch.allclose(result[0][1], original)
# --- Unit Tests: pack_tensors ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackTensors:
"""Test the shared pack_tensors function."""
def test_pack_basic(self):
"""Test packing a few tensors into one buffer."""
params = [
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
("w2", torch.randn(5, dtype=torch.float16).cuda()),
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
assert chunk is not None
assert len(chunk.names) == 2
assert chunk.names == ["w1", "w2"]
assert chunk.shapes == [[10, 20], [5]]
assert chunk.dtypes == [torch.float32, torch.float16]
assert chunk.packed_tensor.dtype == torch.uint8
def test_pack_respects_buffer_limit(self):
"""Test that packing stops when buffer_size_bytes is exceeded."""
params = [
(f"w{i}", torch.randn(100, 100, dtype=torch.float32).cuda())
for i in range(10)
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=50_000,
)
assert chunk is not None
assert len(chunk.names) < 10
def test_pack_empty_iterator(self):
"""Test that an empty iterator returns None."""
chunk = pack_tensors(
iterator=iter([]),
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
assert chunk is None
def test_pack_single_tensor_larger_than_buffer_warns(self):
"""Test that a tensor exceeding buffer_size_bytes emits a warning."""
big = torch.randn(1000, 1000, dtype=torch.float32).cuda()
params = [("big", big)]
import warnings
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter("always")
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=100,
)
assert chunk is not None
assert len(chunk.names) == 1
assert any("exceeds buffer_size_bytes" in str(wi.message) for wi in w)
def test_pack_unpack_roundtrip(self):
"""Test pack then unpack produces identical tensors."""
params = [
("a", torch.randn(8, 16, dtype=torch.float32).cuda()),
("b", torch.randn(4, dtype=torch.float16).cuda()),
("c", torch.randn(3, 5, 7, dtype=torch.bfloat16).cuda()),
]
chunk = pack_tensors(
iterator=iter(params),
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
assert chunk is not None
result = unpack_tensor(
chunk.packed_tensor,
chunk.names,
chunk.shapes,
chunk.dtypes,
chunk.tensor_sizes,
)
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor, rtol=1e-4, atol=1e-6)
def test_pack_multiple_chunks(self):
"""Test consuming an iterator across multiple pack_tensors calls."""
params = [
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
]
it = iter(params)
all_names = []
chunks = []
while True:
chunk = pack_tensors(it, lambda x: x[1], buffer_size_bytes=12_000)
if chunk is None:
break
chunks.append(chunk)
all_names.extend(chunk.names)
assert len(chunks) > 1
assert all_names == [f"w{i}" for i in range(6)]
# --- Unit Tests: packed_ipc_producer ---
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedIpcProducer:
"""Test the packed_ipc_producer generator."""
def test_producer_yields_chunks(self):
"""Test that the producer yields PackedIpcChunk objects."""
params = [
(f"w{i}", torch.randn(50, 50, dtype=torch.float32).cuda()) for i in range(6)
]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="test-uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=12_000,
)
)
assert len(chunks) > 1
def test_producer_ipc_handle_has_uuid(self):
"""Test that each chunk's ipc_handle is keyed by the given UUID."""
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="my-gpu-uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
assert "my-gpu-uuid" in chunks[0].ipc_handle
def test_producer_dtype_names_are_strings(self):
"""Test that dtype_names are string representations."""
params = [
("a", torch.randn(10, dtype=torch.float32).cuda()),
("b", torch.randn(10, dtype=torch.float16).cuda()),
]
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid="uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
assert chunks[0].dtype_names == ["float32", "float16"]
def test_producer_empty_iterator(self):
"""Test producer with empty iterator yields nothing."""
chunks = list(
packed_ipc_producer(
iterator=iter([]),
gpu_uuid="uuid",
post_iter_func=lambda x: x[1],
buffer_size_bytes=1000,
)
)
assert len(chunks) == 0
# --- Integration Tests: IPC Producer-Consumer Roundtrip ---
def _ipc_consumer_worker(cmd_q, ack_q, result_q, done_event, device_index):
"""Worker that consumes chunks streamed one at a time from the parent.
CUDA IPC requires the consumer to be in a separate process from the
producer. The producer reuses a single IPC buffer between chunks, so
the parent must wait for our ack (sent after we copy the chunk to
CPU) before advancing the producer.
"""
try:
torch.accelerator.set_device_index(device_index)
all_results = []
while True:
cd = cmd_q.get()
if cd is None:
break
result = packed_ipc_consumer(
ipc_handle=cd["ipc_handle"],
names=cd["names"],
shapes=cd["shapes"],
dtype_names=cd["dtype_names"],
tensor_sizes=cd["tensor_sizes"],
device_index=device_index,
)
# .cpu() forces a GPU→CPU copy off the shared IPC buffer, so
# the producer is free to overwrite it once we ack.
all_results.extend([(name, tensor.cpu()) for name, tensor in result])
del result
ack_q.put("ack")
result_q.put(("ok", all_results))
except Exception as e:
result_q.put(("error", str(e)))
# Keep the process alive until the parent has finished reading from
# the result queue — torch serializes CPU tensors via fd sharing,
# which requires this process's resource-sharer server to be running.
done_event.wait(timeout=60)
@pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA not available")
class TestPackedIpcRoundtrip:
"""Test IPC producer-consumer roundtrip using real CUDA IPC.
These tests spawn a child process for the consumer because
rebuild_cuda_tensor requires a separate process from the one that
called reduce_tensor.
"""
def _get_gpu_uuid(self) -> str:
device_index = torch.cuda.current_device()
props = torch.cuda.get_device_properties(device_index)
return str(props.uuid)
def _run_roundtrip(self, chunk_iter, device_index, timeout=30):
"""Stream chunks through a child consumer one at a time.
``packed_ipc_producer`` reuses a single IPC buffer for every
chunk, so the producer must not be advanced until the consumer
has finished reading the current chunk. We enforce that with an
ack queue: the consumer puts ``"ack"`` after it has copied the
chunk to CPU, and only then do we pull the next chunk from the
generator.
Returns ``(num_chunks, results)``.
"""
import multiprocessing as mp
ctx = mp.get_context("spawn")
cmd_q = ctx.Queue()
ack_q = ctx.Queue()
result_q = ctx.Queue()
done_event = ctx.Event()
proc = ctx.Process(
target=_ipc_consumer_worker,
args=(cmd_q, ack_q, result_q, done_event, device_index),
)
proc.start()
num_chunks = 0
try:
for chunk in chunk_iter:
cmd_q.put(
{
"ipc_handle": chunk.ipc_handle,
"names": chunk.names,
"shapes": chunk.shapes,
"dtype_names": chunk.dtype_names,
"tensor_sizes": chunk.tensor_sizes,
}
)
if ack_q.get(timeout=timeout) != "ack":
raise RuntimeError("Consumer did not ack chunk")
num_chunks += 1
cmd_q.put(None)
status, payload = result_q.get(timeout=timeout)
finally:
done_event.set()
proc.join(timeout=10)
if proc.is_alive():
proc.kill()
if status == "error":
raise RuntimeError(f"Consumer process failed: {payload}")
# Reclaim IPC-shared memory now that the child has released it
torch.cuda.ipc_collect()
return num_chunks, payload
def test_roundtrip_basic(self):
"""Test basic IPC producer -> consumer roundtrip."""
params = [
("w1", torch.randn(10, 20, dtype=torch.float32).cuda()),
("scale", torch.tensor(1.5, dtype=torch.float32).cuda()),
("w2", torch.randn(5, dtype=torch.float16).cuda()),
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
num_chunks, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
assert num_chunks == 1
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_roundtrip_dtypes(self, dtype):
"""Test IPC roundtrip with different dtypes."""
params = create_mock_model_params(num_layers=2, dtype=dtype)
params_cuda = [(n, t.cuda()) for n, t in params]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
_, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params_cuda),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
assert len(result) == len(params_cuda)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(
params_cuda, result
):
assert orig_name == res_name
assert res_tensor.dtype == dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
def test_roundtrip_multiple_chunks(self):
"""Test IPC roundtrip across multiple chunks."""
params = [
(f"layer{i}.weight", torch.randn(100, 100, dtype=torch.float32).cuda())
for i in range(8)
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
num_chunks, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=50_000,
),
device_index,
)
assert num_chunks > 1
assert len(result) == len(params)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-5, atol=1e-7)
def test_roundtrip_non_contiguous(self):
"""Test IPC roundtrip with non-contiguous tensors."""
params = [
("transposed", torch.randn(20, 10, dtype=torch.float32).cuda().T),
("sliced", torch.randn(40, 30, dtype=torch.float16).cuda()[::2, ::2]),
]
gpu_uuid = self._get_gpu_uuid()
device_index = torch.cuda.current_device()
for _, t in params:
assert not t.is_contiguous()
_, result = self._run_roundtrip(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
),
device_index,
)
for (orig_name, orig_tensor), (res_name, res_tensor) in zip(params, result):
assert orig_name == res_name
assert res_tensor.shape == orig_tensor.shape
assert res_tensor.dtype == orig_tensor.dtype
assert torch.allclose(res_tensor, orig_tensor.cpu(), rtol=1e-4, atol=1e-6)
def test_consumer_wrong_uuid_raises(self):
"""Test that consumer raises ValueError for unknown GPU UUID."""
params = [("w", torch.randn(10, dtype=torch.float32).cuda())]
gpu_uuid = self._get_gpu_uuid()
chunks = list(
packed_ipc_producer(
iterator=iter(params),
gpu_uuid=gpu_uuid,
post_iter_func=lambda x: x[1],
buffer_size_bytes=10_000_000,
)
)
c = chunks[0]
fake_handle = {"fake-uuid-12345": c.ipc_handle[gpu_uuid]}
with pytest.raises(ValueError, match="IPC handle not found"):
packed_ipc_consumer(
ipc_handle=fake_handle,
names=c.names,
shapes=c.shapes,
dtype_names=c.dtype_names,
tensor_sizes=c.tensor_sizes,
device_index=torch.cuda.current_device(),
)
@pytest.mark.parametrize("dtype", [torch.float32, torch.float16, torch.bfloat16])
def test_pack_unpack_scalar(dtype):
"""Preserve scalar shape, dtype and value alongside a matrix on CPU."""
params = [
("weight", torch.randn(2, 3, dtype=dtype)),
("scale", torch.tensor(1.5, dtype=dtype)),
]
chunk = pack_tensors(iter(params), lambda item: item[1], buffer_size_bytes=1024)
assert chunk is not None
result = unpack_tensor(
chunk.packed_tensor,
chunk.names,
chunk.shapes,
chunk.dtypes,
chunk.tensor_sizes,
)
for (expected_name, expected), (name, actual) in zip(params, result, strict=True):
assert name == expected_name
assert actual.shape == expected.shape
assert actual.dtype == expected.dtype
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
@@ -1,628 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the sharded-RDT trainer-side engine.
The trainer engine's own surface: init info and factory registration, the
send_weights round trip against an in-process fake producer server, the
ownership table it resolves and ships to the workers, and the producer's packed
destination-view cache.
The plan/replay half (including `RdtRouter`, which is consumer-side) lives in
test_sharded_rdt_plan.py; the producer server's protocol in
test_sharded_rdt_producer.py.
"""
import itertools
from dataclasses import asdict
import pytest
import torch
from tests.distributed.test_weight_transfer import RecordingClient
from vllm.distributed.weight_transfer import (
ModuleSource,
ParamMeta,
WeightSource,
WeightTransferTrainerFactory,
)
from vllm.distributed.weight_transfer.base import layerwise_groups
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
ShardedRDTTrainerInitInfo,
ShardedRDTTrainerWeightTransferEngine,
)
class _ListSource(WeightSource):
"""A WeightSource over an explicit ordered (name, cpu-tensor) list, so the
sharded-RDT group/order logic can be tested without a real model."""
def __init__(self, pairs):
self._pairs = list(pairs)
def metadata(self):
return [ParamMeta(n, t.dtype, tuple(t.shape)) for n, t in self._pairs]
def __iter__(self):
return iter(self._pairs)
class _FakeProducerServer:
"""In-process stand-in for the _RDTProducerServer Ray actor. Records the
engine->server call sequence and, by default, frees each group as soon as
it is published (simulating the consumers' free_group barrier) so the
gather loop's credit gate never parks. Mirrors the real server's
per-group barrier: publish/free are keyed by GROUP INDEX, signals count
to the ``begin_sync`` live total, publish_group returns nothing, and freed
groups flow back only through wait_freed / end_sync (see
test_sharded_rdt_producer.TestFakeServerAgreesWithTheRealOne)."""
def __init__(self, auto_free=True):
self.order: list[str] = []
self.published: list[int] = []
self.live_count = 1
self._inflight_groups: list[int] = []
self.auto_free = auto_free
self.free_counts: dict[int, int] = {}
self._pending_freed: list[int] = []
# Alias for tests that inspect in-flight groups.
@property
def inflight(self):
return self._inflight_groups
def begin_sync(self, live_count, live_consumer_ids=None):
self.order.append("begin")
self.live_count = max(1, int(live_count))
self.live_ids = live_consumer_ids
def publish_group(self, group_idx, entries):
self.order.append("publish")
self.published.append(group_idx)
self._inflight_groups.append(group_idx)
if self.auto_free or self.free_counts.get(group_idx, 0) >= self.live_count:
self._inflight_groups.remove(group_idx)
self._pending_freed.append(group_idx)
def free_group(self, group_idx):
"""Consumer back-edge; may arrive before the group's publish."""
self.free_counts[group_idx] = self.free_counts.get(group_idx, 0) + 1
if (
group_idx in self._inflight_groups
and self.free_counts[group_idx] >= self.live_count
):
self._inflight_groups.remove(group_idx)
self._pending_freed.append(group_idx)
def wait_freed(self):
"""The engine's credit gate. The real server blocks here; the fake
must already have a banked credit when the gate asks (auto_free, or a
test's own free_group) — anything else is the deadlock the real
watchdog would kill, so fail loudly. Not appended to ``order``: it is
pacing, not a lifecycle milestone."""
assert self._pending_freed, (
"wait_freed with nothing freed: the gather loop would deadlock"
)
freed = self._pending_freed
self._pending_freed = []
return freed
def end_sync(self):
self.order.append("end")
freed = self._pending_freed
self._pending_freed = []
return freed
def set_gather_error(self, message):
self.order.append("error")
def _rdt_engine_with_fake_server(
source, *, is_sender, client, server, monkeypatch, fleet_owned=None
):
"""Build a ShardedRDTTrainerWeightTransferEngine wired to an in-process fake
server (no Ray, no CUDA IPC): bypass trainer_init's spawn, set the
group-major metadata, and route _rpc to the fake."""
import vllm.distributed.weight_transfer.sharded_rdt_trainer as mod
# reduce_tensor needs CUDA; the fake server never rebuilds, so stub it.
monkeypatch.setattr(mod, "reduce_tensor", lambda t: (None, ("fake",)))
init_info = ShardedRDTTrainerInitInfo(num_consumers=1, rank=0 if is_sender else 1)
engine = ShardedRDTTrainerWeightTransferEngine(
client=client, source=source, is_sender=is_sender, init_info=init_info
)
engine._meta = list(source.metadata())
names = [m.name for m in engine._meta]
engine._groups = layerwise_groups(names)
engine._server = server
engine._rpc = lambda method, *args: getattr(server, method)(*args)
# What trainer_init resolves from the source's held names + the fleet's
# all-gather. ``fleet_owned`` stands in for that all-gather so a partial-
# ownership rank can be tested without a real process group; the fleet must
# cover every name or the transpose rejects it (nothing would serve the rest).
if fleet_owned is None:
engine._resolve_ownership(1, 0)
else:
# Stands in for the (metadata digest, held-name bitmask) all-gather:
# each rank's group list becomes the mask its held names would set.
names = [m.name for m in engine._meta]
groups = engine._groups
def _fake_gather(_world, mine):
out = []
for owned in fleet_owned:
held = {n for gi in owned for n in groups[gi]}
mask = bytearray((len(names) + 7) // 8)
for i, n in enumerate(names):
if n in held:
mask[i >> 3] |= 1 << (i & 7)
out.append((mine[0], bytes(mask)))
return out
monkeypatch.setattr(engine, "_all_gather_owned", _fake_gather)
engine._resolve_ownership(len(fleet_owned), 0)
return engine
def _rdt_source_two_layers():
return _ListSource(
[
("embed.weight", torch.zeros(2)),
("model.layers.0.w", torch.zeros(2)),
("model.layers.1.w", torch.zeros(2)),
("norm.weight", torch.zeros(2)),
]
)
class TestShardedRDTTrainerInitInfo:
def test_declares_backend(self):
assert ShardedRDTTrainerInitInfo.backend == "sharded_rdt"
def test_rank_is_keyword_only_and_drives_is_sender(self):
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=0).is_sender is True
assert ShardedRDTTrainerInitInfo(num_consumers=4, rank=1).is_sender is False
with pytest.raises(TypeError):
# rank is keyword-only.
ShardedRDTTrainerInitInfo(4, 0) # type: ignore[misc]
def test_registered_in_trainer_factory(self):
cls = WeightTransferTrainerFactory._registry["sharded_rdt"]()
assert cls is ShardedRDTTrainerWeightTransferEngine
def test_sharded_rdt_trainer_init_requires_source():
with pytest.raises(ValueError, match="requires a WeightSource"):
ShardedRDTTrainerWeightTransferEngine.trainer_init(
ShardedRDTTrainerInitInfo(num_consumers=1, rank=0),
client=RecordingClient(),
source=None,
)
def test_sharded_rdt_worker_init_info_is_group_major(monkeypatch):
source = _rdt_source_two_layers()
engine = _rdt_engine_with_fake_server(
source,
is_sender=True,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
)
worker_init = engine._build_worker_init_info(["srv_rk0"])
# 4 params -> pre / layer0 / layer1 / post = 4 groups of length 1.
assert worker_init.names == [
"embed.weight",
"model.layers.0.w",
"model.layers.1.w",
"norm.weight",
]
assert worker_init.group_lens == [1, 1, 1, 1]
assert worker_init.trainer_actor_names == ["srv_rk0"]
assert worker_init.produce_method_name == "rdt_produce_weights_batched"
assert sum(worker_init.group_lens) == len(worker_init.names)
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_send_weights_drives_client_in_order(monkeypatch):
server = _FakeProducerServer(auto_free=True)
client = RecordingClient()
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=True,
client=client,
server=server,
monkeypatch=monkeypatch,
)
engine.send_weights()
assert client.order == ["start", "update", "finish"]
# begin, one publish per group (4), end.
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
assert len(server.published) == 4
# every group freed -> no engine-held refs remain.
assert engine._inflight == {}
def test_sharded_rdt_send_weights_group_order_mismatch_raises(monkeypatch):
# Source whose iteration order disagrees with its metadata order.
class _BadSource(_ListSource):
def __iter__(self):
reordered = list(self._pairs)
reordered[0], reordered[1] = reordered[1], reordered[0]
return iter(reordered)
server = _FakeProducerServer()
engine = _rdt_engine_with_fake_server(
_BadSource(_rdt_source_two_layers()._pairs),
is_sender=True,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
)
with pytest.raises(RuntimeError, match="iteration order must match"):
engine.send_weights()
assert "error" in server.order # gather error propagated to the server
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_non_sender_skips_client(monkeypatch):
class _RaisingClient(RecordingClient):
def start_weight_update(self):
raise AssertionError("non-sender must not touch the client")
def update_weights(self, update_info):
raise AssertionError("non-sender must not touch the client")
def finish_weight_update(self, weight_version: str | None = None):
raise AssertionError("non-sender must not touch the client")
server = _FakeProducerServer(auto_free=True)
client = _RaisingClient()
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=False,
client=client,
server=server,
monkeypatch=monkeypatch,
)
engine.send_weights() # gathers only; must not raise
assert client.order == []
assert server.order == ["begin", "publish", "publish", "publish", "publish", "end"]
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_send_weights_surfaces_update_error(monkeypatch):
class _FailingUpdateClient(RecordingClient):
def update_weights(self, update_info):
self.order.append("update")
raise RuntimeError("inference side rejected update")
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=True,
client=_FailingUpdateClient(),
server=server,
monkeypatch=monkeypatch,
)
with pytest.raises(RuntimeError, match="inference side rejected update"):
engine.send_weights()
class _OwnedSource(_ListSource):
"""A source holding only some groups' names, like a pipeline-parallel rank.
Takes group indices for convenience and declares the names inside them."""
def __init__(self, pairs, owned_group_idx):
super().__init__(pairs)
self._owned = list(owned_group_idx)
groups = layerwise_groups([n for n, _ in pairs])
self._owned_names = [n for gi in self._owned for n in groups[gi]]
def held_names(self):
return list(self._owned_names)
def __iter__(self):
by_name = dict(self._pairs)
return iter([(n, by_name[n]) for n in self._owned_names])
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_publishes_only_the_groups_it_holds(monkeypatch):
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
fleet_owned=[[1, 2], [0, 3]], # this rank holds the layers, rank 1 the rest
)
engine.send_weights()
assert server.published == [1, 2]
assert server.order == ["begin", "publish", "publish", "end"]
assert engine._inflight == {}
# embed + norm on rank 1, the two layers here: two distinct owner sets,
# numbered by first appearance in metadata order.
assert engine._owner_sets == [[1], [0]]
assert engine._name_owner_class == [0, 1, 1, 0]
def test_sharded_rdt_owned_group_order_mismatch_raises(monkeypatch):
class _MisorderedOwned(_OwnedSource):
def __iter__(self):
return iter(list(super().__iter__())[::-1])
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_MisorderedOwned(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
fleet_owned=[[1, 2], [0, 3]],
)
with pytest.raises(RuntimeError, match="iteration order must match"):
engine.send_weights()
assert "error" in server.order
@pytest.mark.skipif(
not torch.cuda.is_available(),
reason="the gather loop's CUDA-IPC export needs a device",
)
def test_sharded_rdt_begin_sync_carries_the_live_count(monkeypatch):
"""The free barrier's target is one integer per sync: the live consumer
count, defaulting to the whole provisioned fleet."""
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_rdt_source_two_layers(),
is_sender=False,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
)
engine.send_weights()
assert server.live_count == 1 # num_consumers=1 in the harness
engine.send_weights(live_consumer_ids=[0])
assert server.live_count == 1
class TestLiveCountPlumbing:
"""``send_weights(live_consumer_ids)`` -> the barrier target. The
provisioned geometry is frozen; a degraded sync only lowers the target and
narrows each slot-sharing group's rendezvous."""
@staticmethod
def _engine(num_consumers, world=2, rank=0):
engine = ShardedRDTTrainerWeightTransferEngine.__new__(
ShardedRDTTrainerWeightTransferEngine
)
engine._init_info = ShardedRDTTrainerInitInfo(
rank=rank, num_consumers=num_consumers
)
engine.source = object() # send_weights asserts a source is present
received: list = []
engine._send_weights_inner = lambda count, ids: received.append((count, ids))
return engine, received
def test_none_counts_the_whole_provisioned_fleet(self):
engine, got = self._engine(num_consumers=8)
engine.send_weights(None)
assert got == [(8, list(range(8)))]
def test_a_live_set_counts_its_distinct_members(self):
engine, got = self._engine(num_consumers=8)
engine.send_weights([0, 1, 4, 5, 5])
assert got == [(4, [0, 1, 4, 5])]
def test_a_full_live_set_matches_the_provisioned_count(self):
engine, got = self._engine(num_consumers=8)
engine.send_weights(list(range(8)))
engine.send_weights(None)
assert got == [(8, list(range(8)))] * 2
def test_which_consumers_died_does_not_matter_only_how_many(self):
"""The whole point of the barrier: no routed per-producer targets, so
the identity of the dead consumer is irrelevant to the producers."""
counts = []
for live in ([0, 1, 2, 3], [4, 5, 6, 7], [0, 2, 4, 6]):
engine, got = self._engine(num_consumers=8)
engine.send_weights(live)
counts += [c for c, _ids in got]
assert counts == [4, 4, 4]
def test_the_live_ids_travel_with_the_count(self):
"""The count sizes the free barrier, the ids size the slot-sharing
rendezvous, so a producer that shares slots can tell WHICH consumers it
is still waiting for. They must describe the same set."""
engine, got = self._engine(num_consumers=8)
engine.send_weights([6, 0, 2, 2])
((count, ids),) = got
assert ids == [0, 2, 6] and count == len(ids)
def test_sharded_rdt_worker_init_info_carries_the_ownership_table(monkeypatch):
import json
server = _FakeProducerServer(auto_free=True)
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
is_sender=True,
client=RecordingClient(),
server=server,
monkeypatch=monkeypatch,
fleet_owned=[[0, 1], [2, 3]],
)
worker_init = engine._build_worker_init_info(["srv_rk0", "srv_rk1"])
assert worker_init.owner_sets == [[0], [1]]
assert worker_init.name_owner_class == [0, 0, 1, 1]
assert len(worker_init.name_owner_class) == len(worker_init.names)
# The payload crosses the control plane as JSON: nested lists must survive.
round_tripped = json.loads(json.dumps(asdict(worker_init)))
assert round_tripped["owner_sets"] == [[0], [1]]
assert round_tripped["name_owner_class"] == [0, 0, 1, 1]
def test_weight_source_holds_everything_by_default():
"""The contract's default: a source that says nothing holds the whole model."""
src = _rdt_source_two_layers()
assert src.held_names() is None
assert ModuleSource(torch.nn.Linear(2, 2)).held_names() is None
def test_sharded_rdt_rejects_a_held_name_outside_metadata(monkeypatch):
class _BadHeld(_OwnedSource):
def held_names(self):
return ["embed.weight", "not.a.real.name"]
with pytest.raises(ValueError, match="not"):
_rdt_engine_with_fake_server(
_BadHeld(_rdt_source_two_layers()._pairs, [0]),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
)
def test_sharded_rdt_rejects_a_rank_holding_nothing(monkeypatch):
class _HoldsNothing(_ListSource):
def held_names(self):
return []
with pytest.raises(ValueError, match="empty"):
_rdt_engine_with_fake_server(
_HoldsNothing(_rdt_source_two_layers()._pairs),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
)
def test_sharded_rdt_rejects_a_name_no_rank_holds(monkeypatch):
"""Every name must be held somewhere or it can never be served — caught
when the holdings are transposed, naming the orphan."""
with pytest.raises(ValueError, match="no trainer rank holds"):
_rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [0, 1]),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(),
monkeypatch=monkeypatch,
fleet_owned=[[0, 1], [2]], # nobody holds group 3
)
def test_sharded_rdt_rejects_metadata_disagreement_across_ranks(monkeypatch):
"""Only the sender's metadata reaches the consumers, so a rank describing
just its own share must fail loudly rather than silently drop the rest."""
engine = _rdt_engine_with_fake_server(
_OwnedSource(_rdt_source_two_layers()._pairs, [1, 2]),
is_sender=False,
client=RecordingClient(),
server=_FakeProducerServer(auto_free=True),
monkeypatch=monkeypatch,
fleet_owned=[[1, 2], [0, 3]], # a covering fleet, so construction succeeds
)
# Now rank 1 reports a DIFFERENT metadata digest for the same model.
monkeypatch.setattr(
engine,
"_all_gather_owned",
lambda w, mine: [mine, ("deadbeefdeadbeef", mine[1])],
)
with pytest.raises(ValueError, match="disagrees across trainer ranks"):
engine._resolve_ownership(2, 0)
def _serve_ring_server(src_name, src):
"""A producer server with one cached tensor and a pre-seeded serve ring, so a
pull needs no Ray and no NIXL registration. Returns (server, serve) where
``serve(chain)`` packs one spec into the SAME ring slot every time — which is
what puts the destination-view cache, and only it, under test."""
from vllm.distributed.weight_transfer.sharded_rdt_trainer import (
_RDTProducerServer,
)
srv = _RDTProducerServer(
num_rdt_buffers=2,
buffer_presize_gb=0.0,
gather_lookahead=2,
)
srv._cache[src_name] = src
srv._serve_rings[0] = [
torch.empty(1 << 16, dtype=torch.uint8, device="cuda") for _ in range(2)
]
# The slot is ``seq % nring``, so stepping seq by nring keeps every pull in
# slot 0 while still making each one its own generation -- reusing a seq
# would rendezvous with the finished pack and return it unchanged.
seqs = itertools.count(0, srv._nring)
def serve(chain):
return srv.rdt_produce_weights_batched(
[(src_name, chain)], consumer_id=0, seq=next(seqs)
)[0]
return srv, serve
@pytest.mark.skipif(
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
)
def test_serve_does_not_reuse_packed_views_of_another_shape():
"""Two requests can share a name yet pack different slices of it.
The producer caches the destination views it carves into a serve ring slot.
Keyed by name alone, the second request is packed through the first's views.
Reachable when one name's copies split across owner-class chunks.
"""
name = "model.layers.0.w"
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
_srv, serve = _serve_ring_server(name, src)
serve((("narrow", (0, 0, 2), ()),)) # 2 rows
wide = src.narrow(0, 0, 6) # 6 rows, same name, same slot
blob = serve((("narrow", (0, 0, 6), ()),))
got = blob[: wide.numel() * wide.element_size()].view(wide.dtype).reshape(6, 8)
assert torch.equal(got, wide)
@pytest.mark.skipif(
not torch.cuda.is_available(), reason="the producer server needs a CUDA device"
)
def test_serve_does_not_reuse_packed_views_of_another_dtype():
"""The SILENT case of the same cache: two requests whose slices have the same
name and the same shape but different dtypes pack at identical offsets, so
reusing the stale views raises nothing — ``copy_`` just casts, and the blob
carries the wrong bytes with no check downstream.
``view(dtype)`` rather than ``to(dtype)`` because the chain is replayed under
``ALLOWED_OPS``, which rejects ``to``: same-itemsize dtypes keep the byte
length identical, which is exactly what makes the stale view fit.
"""
name = "model.layers.0.w"
src = torch.arange(64, dtype=torch.bfloat16, device="cuda").reshape(8, 8)
_srv, serve = _serve_ring_server(name, src)
serve((("view", (torch.float16,), ()),)) # same shape and bytes, fp16
blob = serve(()) # same name and shape, bf16
got = blob[: src.numel() * src.element_size()].view(src.dtype).reshape(8, 8)
assert torch.equal(got, src), "packed through a cached view of the wrong dtype"
File diff suppressed because it is too large Load Diff
@@ -1,465 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Tests for the pre-shared ncclUniqueId weight-transfer rendezvous.
The rendezvous lets a peer that cannot join a TCPStore (e.g. a torch-free JAX
trainer) initialize NCCL weight transfer from a unique id minted out of band.
* Unit tests for the mutually-exclusive rendezvous-mode validation on the NCCL
init infos.
* Two-GPU integration tests driving the real worker engine against a peer: one
via vLLM's own ``PyNcclCommunicator``, one via a genuinely torch-free trainer
(cupy buffer + raw ``ncclBroadcast``).
The shared Ray helpers come from the sibling module (unchanged); the worker task
is local so the sibling's TCP test and its worker stay untouched.
"""
from unittest.mock import MagicMock
import pybase64 as base64
import pytest
import ray
import torch
from tests.distributed.test_weight_transfer import (
_init_ray_for_weight_transfer,
_set_ray_assigned_device,
create_mock_vllm_config,
)
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerInitInfo,
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
)
# Standard base64 of exactly 128 zero bytes: a well-formed (dummy) ncclUniqueId.
VALID_UID_B64 = base64.b64encode(b"\x00" * 128).decode()
class TestNCCLRendezvousValidation:
"""Test the xor-of-rendezvous-modes validation on the NCCL init infos."""
def test_worker_init_info_is_keyword_only(self):
# kw_only so a stale positional call fails loudly rather than silently
# swapping fields once the rendezvous fields became optional.
with pytest.raises(TypeError):
NCCLWeightTransferInitInfo(1, 3, "127.0.0.1", 12345) # type: ignore[call-arg]
def test_trainer_init_info_accepts_positional_fields(self):
# NCCLTrainerInitInfo keeps positional subclass fields; only `rank`
# (from TrainerInitInfo) is keyword-only.
trainer_positional = NCCLTrainerInitInfo("127.0.0.1", 12345, 3, rank=0)
trainer_keyword = NCCLTrainerInitInfo(
master_address="127.0.0.1",
master_port=12345,
world_size=3,
rank=0,
)
assert trainer_positional == trainer_keyword
def test_tcp_mode_valid(self):
info = NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
master_port=12345,
)
assert info.master_address == "127.0.0.1"
assert info.nccl_unique_id_b64 is None
def test_uid_mode_valid(self):
info = NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64=VALID_UID_B64,
)
assert info.nccl_unique_id_b64 == VALID_UID_B64
assert info.master_address is None
assert info.master_port is None
assert VALID_UID_B64 not in repr(info)
def test_both_modes_raises(self):
with pytest.raises(ValueError, match="not both"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
master_port=12345,
nccl_unique_id_b64=VALID_UID_B64,
)
def test_neither_mode_raises(self):
with pytest.raises(ValueError, match="need nccl_unique_id_b64"):
NCCLWeightTransferInitInfo(rank_offset=1, world_size=3)
def test_half_specified_tcp_raises(self):
with pytest.raises(ValueError, match="master_port"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
master_address="127.0.0.1",
)
def test_bad_base64_uid_raises(self):
with pytest.raises(ValueError, match="not valid standard base64"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64="!!!!",
)
def test_wrong_length_uid_raises(self):
short_uid = base64.b64encode(b"\x00" * 64).decode()
with pytest.raises(ValueError, match="128"):
NCCLWeightTransferInitInfo(
rank_offset=1,
world_size=3,
nccl_unique_id_b64=short_uid,
)
def test_parse_init_info_uid_valid(self):
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda"),
MagicMock(spec=torch.nn.Module),
)
init_info = engine.parse_init_info(
{
"nccl_unique_id_b64": VALID_UID_B64,
"rank_offset": 1,
"world_size": 3,
}
)
assert isinstance(init_info, NCCLWeightTransferInitInfo)
assert init_info.nccl_unique_id_b64 == VALID_UID_B64
assert init_info.master_address is None
def test_parse_init_info_both_modes_raises(self):
engine = NCCLWeightTransferEngine(
WeightTransferConfig(backend="nccl"),
create_mock_vllm_config(),
torch.device("cuda"),
MagicMock(spec=torch.nn.Module),
)
with pytest.raises(ValueError, match="not both"):
engine.parse_init_info(
{
"master_address": "127.0.0.1",
"master_port": 12345,
"nccl_unique_id_b64": VALID_UID_B64,
"rank_offset": 1,
"world_size": 3,
}
)
@pytest.mark.parametrize(
("unique_id_bytes", "rank", "world_size", "match"),
[
(b"\x00" * 64, 0, 2, "128-byte"),
(b"\x00" * 128, -1, 2, "out of range"),
(b"\x00" * 128, 2, 2, "out of range"),
],
)
def test_from_unique_id_bytes_validation(
self, unique_id_bytes, rank, world_size, match
):
from vllm.distributed.device_communicators.pynccl import PyNcclCommunicator
with pytest.raises(ValueError, match=match):
PyNcclCommunicator.from_unique_id_bytes(
unique_id_bytes,
rank=rank,
world_size=world_size,
device=0,
)
@ray.remote(num_gpus=1)
def inference_receive_tensor(
init_info_dict: dict,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Worker that joins via the given init dict and records the tensor it gets.
Local to this module (rather than the sibling's TCP-only worker) so it can
take a UID init payload -- the same dict boundary an external trainer POSTs.
"""
import contextlib
import torch
_set_ray_assigned_device()
from vllm.config.parallel import ParallelConfig
from vllm.config.weight_transfer import WeightTransferConfig
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLWeightTransferEngine,
NCCLWeightTransferInitInfo,
NCCLWeightTransferUpdateInfo,
)
class Recorder(torch.nn.Module):
def __init__(self):
super().__init__()
self.received = []
def load_weights(self, weights):
for name, tensor in weights:
self.received.append((name, tensor.clone()))
config = WeightTransferConfig(backend="nccl")
vllm_config = MagicMock()
parallel_config = MagicMock(spec=ParallelConfig)
parallel_config.rank = 0
parallel_config.world_size = 1
parallel_config.data_parallel_rank = 0
parallel_config.data_parallel_index = 0
vllm_config.parallel_config = parallel_config
vllm_config.model_config = MagicMock()
recorder = Recorder()
engine = NCCLWeightTransferEngine(
config, vllm_config, torch.device("cuda"), recorder
)
# Transport-only test: bypass the set_current_vllm_config context that
# receive_weights enters, since vllm_config here is a mock.
import vllm.config as _vllm_config_mod
_vllm_config_mod.set_current_vllm_config = lambda cfg: contextlib.nullcontext()
try:
engine.init_transfer_engine(NCCLWeightTransferInitInfo(**init_info_dict))
engine.receive_weights(
NCCLWeightTransferUpdateInfo(
names=["test.weight"],
dtype_names=[tensor_dtype],
shapes=[tensor_shape],
)
)
torch.accelerator.synchronize()
received_shape = None
received_sum = None
if len(recorder.received) == 1:
_name, tensor = recorder.received[0]
received_shape = list(tensor.shape)
received_sum = tensor.sum().item()
return {
"success": received_shape == tensor_shape
and abs(received_sum - torch.tensor(tensor_shape).prod().item()) < 0.01,
"received_shape": received_shape,
"received_sum": received_sum,
}
finally:
engine.shutdown()
@ray.remote(num_gpus=1)
def trainer_broadcast_tensor_uid(
uid_b64: str,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Drive rank 0 via vLLM's own communicator from a pre-shared unique id.
A real torch-free trainer would mint the id and drive rank 0 itself; here we
reuse the real send protocol by injecting the communicator into a sender
engine, exercising the worker-side UID path end to end.
"""
import pybase64 as base64
import torch
_set_ray_assigned_device()
from vllm.distributed.weight_transfer import ModuleSource
from vllm.distributed.weight_transfer.nccl_common import uid_init_process_group
from vllm.distributed.weight_transfer.nccl_engine import (
NCCLTrainerWeightTransferEngine,
)
class NoopClient:
def init_weight_transfer_engine(self, init_info):
pass
def start_weight_update(self):
pass
def update_weights(self, update_info):
pass
def finish_weight_update(self, weight_version=None):
pass
dtype = getattr(torch, tensor_dtype)
model = torch.nn.Module()
test_module = torch.nn.Module()
test_module.register_parameter(
"weight",
torch.nn.Parameter(
torch.ones(tensor_shape, dtype=dtype, device="cuda"), requires_grad=False
),
)
model.add_module("test", test_module)
engine = NCCLTrainerWeightTransferEngine(
client=NoopClient(),
source=ModuleSource(model),
is_sender=True,
packed=False,
)
engine.model_update_group = uid_init_process_group(
base64.b64decode(uid_b64),
rank=0,
world_size=world_size,
device=torch.accelerator.current_device_index(),
)
try:
engine.send_weights()
torch.accelerator.synchronize()
return {"success": True}
finally:
engine.shutdown()
@ray.remote(num_gpus=1)
def trainer_broadcast_tensor_torch_free(
uid_b64: str,
world_size: int,
tensor_shape: list[int],
tensor_dtype: str,
) -> dict:
"""Trainer peer that uses no torch and no PyNcclCommunicator (a JAX stand-in).
Drives NCCL through the raw ctypes wrapper with a cupy send buffer, so it
exercises interop with a foreign NCCL peer rather than vLLM against itself.
``PyNcclCommunicator.__init__`` runs a one-element warm-up all_reduce right
after ``ncclCommInitRank``; that is a collective, so this peer must issue the
matching all_reduce before the broadcast or both ranks deadlock.
"""
import cupy as cp
import numpy as np
import pybase64 as base64
device = _set_ray_assigned_device()
cp.cuda.Device(device.index).use()
from vllm.distributed.device_communicators.pynccl_wrapper import (
NCCLLibrary,
buffer_type,
cudaStream_t,
ncclDataTypeEnum,
ncclRedOpTypeEnum,
)
assert tensor_dtype == "float32", "harness only wires float32"
# Fill on the host and copy up (a plain H2D memcpy) so we neither use torch
# nor make cupy JIT-compile a fill kernel (which needs NVRTC). All-ones lets
# the worker verify by sum == numel.
sendbuff = cp.asarray(np.ones(tensor_shape, dtype=np.float32))
warmup = cp.asarray(np.zeros(1, dtype=np.float32))
numel = int(sendbuff.size)
# The copies above run on the default stream; drain them before the NCCL
# collectives read the buffers on our own stream.
cp.cuda.Device(device.index).synchronize()
stream = cp.cuda.Stream()
nccl = NCCLLibrary()
comm = nccl.ncclCommInitRank(
world_size, nccl.unique_id_from_bytes(base64.b64decode(uid_b64)), 0
)
try:
# Mirror the worker's init-time warm-up all_reduce (rank 0 must join it).
nccl.ncclAllReduce(
buffer_type(warmup.data.ptr),
buffer_type(warmup.data.ptr),
1,
ncclDataTypeEnum.ncclFloat32,
ncclRedOpTypeEnum.ncclSum,
comm,
cudaStream_t(stream.ptr),
)
nccl.ncclBroadcast(
buffer_type(sendbuff.data.ptr),
buffer_type(sendbuff.data.ptr),
numel,
ncclDataTypeEnum.ncclFloat32,
0, # root
comm,
cudaStream_t(stream.ptr),
)
stream.synchronize()
return {"success": True}
finally:
# Abort (not the collective destroy) for an uncoordinated teardown.
nccl.ncclCommAbort(comm)
def _run_uid_transfer(trainer_task) -> None:
"""Mint a unique id, run the worker and the given trainer peer concurrently,
and assert the tensor round-trips. Both ranks must enter init together --
there is no store barrier on the UID path."""
from vllm.distributed.device_communicators.pynccl_wrapper import NCCLLibrary
_init_ray_for_weight_transfer()
nccl = NCCLLibrary()
uid_b64 = base64.b64encode(bytes(nccl.ncclGetUniqueId().internal)).decode()
world_size = 2 # 1 trainer + 1 inference worker
tensor_shape = [100, 100]
tensor_dtype = "float32"
inference_future = inference_receive_tensor.remote(
{
"rank_offset": 1,
"world_size": world_size,
"nccl_unique_id_b64": uid_b64,
"packed": False,
},
tensor_shape,
tensor_dtype,
)
trainer_future = trainer_task.remote(
uid_b64, world_size, tensor_shape, tensor_dtype
)
try:
trainer_result, result = ray.get(
[trainer_future, inference_future], timeout=120
)
finally:
ray.shutdown()
assert trainer_result["success"], "Trainer should complete successfully"
assert result["success"], (
f"Weight transfer failed. "
f"Received shape: {result['received_shape']}, "
f"Received sum: {result['received_sum']}"
)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_between_processes_uid():
"""Weight transfer over a pre-shared ncclUniqueId (no TCPStore), with both
ranks on vLLM's ``PyNcclCommunicator``."""
_run_uid_transfer(trainer_broadcast_tensor_uid)
@pytest.mark.skipif(
torch.accelerator.device_count() < 2,
reason="Need at least 2 GPUs to run NCCL weight transfer test.",
)
def test_nccl_weight_transfer_torch_free_trainer():
"""UID weight transfer against a foreign peer that uses no torch and no vLLM
communicator, only a cupy buffer and raw ``ncclBroadcast``."""
pytest.importorskip("cupy")
_run_uid_transfer(trainer_broadcast_tensor_torch_free)