[Perf] Zero-copy torch.Tensor pickling in shm_broadcast MessageQueue (#48442)
Signed-off-by: Ruinan Ma <r7ma3088@gmail.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import io
|
||||
import pickle
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
@@ -10,12 +12,15 @@ from unittest import mock
|
||||
import multiprocess as mp
|
||||
import numpy as np
|
||||
import pytest
|
||||
import torch
|
||||
import torch.distributed as dist
|
||||
|
||||
from vllm.distributed.device_communicators import shm_broadcast
|
||||
from vllm.distributed.device_communicators.shm_broadcast import (
|
||||
MessageQueue,
|
||||
ShmRingBuffer,
|
||||
_rebuild_tensor,
|
||||
_reduce_tensor,
|
||||
check_shm_free_space,
|
||||
)
|
||||
from vllm.distributed.utils import StatelessProcessGroup
|
||||
@@ -354,6 +359,150 @@ def test_message_queue_busy_to_idle():
|
||||
distributed_run(worker_fn_test_busy_to_idle, 4)
|
||||
|
||||
|
||||
@worker_fn_wrapper
|
||||
def worker_fn_tensor_broadcast():
|
||||
rank = dist.get_rank()
|
||||
writer_rank = 0
|
||||
message_queue = MessageQueue.create_from_process_group(
|
||||
dist.group.WORLD, 8 * 1024 * 1024, 4, writer_rank
|
||||
)
|
||||
|
||||
# Both ranks construct the identical reference payload.
|
||||
torch.manual_seed(42)
|
||||
payload = {
|
||||
# 2MiB: rides the shm ring as an out-of-band buffer (the receiving
|
||||
# side must copy out of the reusable ring chunk).
|
||||
"mid": torch.randn(1024, 512),
|
||||
# 16MiB > max_chunk_bytes: overflows to the zmq socket (the
|
||||
# receiving side aliases the zmq.Frame zero-copy).
|
||||
"big": torch.randn(4096, 2048, dtype=torch.bfloat16),
|
||||
"nested": ["plain", 123, {"inner": torch.arange(5)}],
|
||||
}
|
||||
|
||||
if rank == writer_rank:
|
||||
with mock.patch(
|
||||
"vllm.distributed.device_communicators.shm_broadcast._reduce_tensor",
|
||||
wraps=_reduce_tensor,
|
||||
) as wrapped_reduce:
|
||||
message_queue.enqueue(payload)
|
||||
assert wrapped_reduce.call_count == 3
|
||||
# Cycle the ring (max_chunks=4) several times over so that aliased
|
||||
# ring chunks would be overwritten.
|
||||
for i in range(16):
|
||||
message_queue.enqueue({"junk": torch.full((1024, 512), float(i))})
|
||||
else:
|
||||
received = message_queue.dequeue(timeout=30)
|
||||
for key in ("mid", "big"):
|
||||
assert torch.equal(received[key], payload[key]), key
|
||||
assert received[key].dtype == payload[key].dtype, key
|
||||
assert torch.equal(received["nested"][2]["inner"], torch.arange(5))
|
||||
|
||||
snapshot = received["mid"].clone()
|
||||
for i in range(16):
|
||||
junk = message_queue.dequeue(timeout=30)
|
||||
assert torch.equal(junk["junk"], torch.full((1024, 512), float(i)))
|
||||
# Tensors received via the shm ring must not alias chunk memory
|
||||
# that the writer has reused for subsequent messages.
|
||||
assert torch.equal(received["mid"], snapshot)
|
||||
# Rebuilt tensors must be writable, like regular tensors.
|
||||
received["mid"] += 1.0
|
||||
received["big"][0, 0] = 1.0
|
||||
|
||||
dist.barrier()
|
||||
print(f"tensor broadcast passed the test! Rank {rank}")
|
||||
|
||||
|
||||
def test_tensor_broadcast():
|
||||
distributed_run(worker_fn_tensor_broadcast, 2)
|
||||
|
||||
|
||||
def _dumps_oob(obj) -> tuple[bytes, list]:
|
||||
"""Pickle `obj` the same way `MessageQueue.enqueue` does: tensor
|
||||
dispatch table + out-of-band buffers >= 1MiB."""
|
||||
buffers = []
|
||||
|
||||
def callback(buf: pickle.PickleBuffer) -> bool:
|
||||
raw = buf.raw()
|
||||
if raw.nbytes < 1024 * 1024:
|
||||
return True
|
||||
buffers.append(raw)
|
||||
return False
|
||||
|
||||
bio = io.BytesIO()
|
||||
pickler = pickle.Pickler(
|
||||
bio, protocol=pickle.HIGHEST_PROTOCOL, buffer_callback=callback
|
||||
)
|
||||
pickler.dispatch_table = {torch.Tensor: _reduce_tensor}
|
||||
pickler.dump(obj)
|
||||
return bio.getvalue(), buffers
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"case",
|
||||
[
|
||||
"small",
|
||||
"mid",
|
||||
"bf16",
|
||||
"fp8",
|
||||
"empty",
|
||||
"scalar",
|
||||
"noncontig",
|
||||
"requires_grad",
|
||||
"conj",
|
||||
"param",
|
||||
],
|
||||
)
|
||||
def test_tensor_pickle_roundtrip(case: str):
|
||||
tensor = {
|
||||
# Inlined in-band (< 1MiB) and out-of-band (>= 1MiB) buffers.
|
||||
"small": lambda: torch.randn(100, 10),
|
||||
"mid": lambda: torch.randn(1024, 512),
|
||||
# Dtypes numpy doesn't recognize.
|
||||
"bf16": lambda: torch.randn(512, 512, dtype=torch.bfloat16),
|
||||
"fp8": lambda: torch.randn(32, 32).to(torch.float8_e4m3fn),
|
||||
# Shape edge cases.
|
||||
"empty": lambda: torch.empty(0, 8),
|
||||
"scalar": lambda: torch.tensor(3.14),
|
||||
"noncontig": lambda: torch.randn(64, 64).t(),
|
||||
# These fall back to torch's default reducer.
|
||||
"requires_grad": lambda: torch.randn(8, 8, requires_grad=True),
|
||||
"conj": lambda: torch.randn(4, dtype=torch.complex64).conj(),
|
||||
"param": lambda: torch.nn.Parameter(torch.randn(4), requires_grad=False),
|
||||
}[case]()
|
||||
|
||||
data, buffers = _dumps_oob({"tensor": tensor, "meta": list(range(10))})
|
||||
received = pickle.loads(data, buffers=buffers)["tensor"]
|
||||
|
||||
assert received.shape == tensor.shape
|
||||
assert received.dtype == tensor.dtype
|
||||
if tensor.dtype == torch.float8_e4m3fn:
|
||||
assert torch.equal(received.view(torch.uint8), tensor.view(torch.uint8))
|
||||
else:
|
||||
assert torch.equal(received, tensor)
|
||||
assert received.requires_grad == tensor.requires_grad
|
||||
assert isinstance(received, type(tensor))
|
||||
if tensor.numel() and not tensor.requires_grad:
|
||||
# Rebuilt tensors must be writable, like regular tensors.
|
||||
received.view(-1)[0] = 1.0
|
||||
|
||||
|
||||
@pytest.mark.parametrize("case", ["cuda", "requires_grad", "conj"])
|
||||
def test_reduce_tensor_fallback(case: str):
|
||||
"""Tensors the zero-copy reducer can't safely alias must fall back to
|
||||
torch's default reduction."""
|
||||
if case == "cuda":
|
||||
if not torch.cuda.is_available():
|
||||
pytest.skip("requires CUDA")
|
||||
tensor = torch.randn(4, device="cuda")
|
||||
elif case == "requires_grad":
|
||||
tensor = torch.randn(8, requires_grad=True)
|
||||
else:
|
||||
tensor = torch.randn(4, dtype=torch.complex64).conj()
|
||||
|
||||
reduced = _reduce_tensor(tensor)
|
||||
assert reduced[0] is not _rebuild_tensor
|
||||
|
||||
|
||||
@pytest.mark.parametrize("should_warn", [False, True])
|
||||
def test_reader_timeout_caps_indefinite_waits(should_warn):
|
||||
with (
|
||||
|
||||
Reference in New Issue
Block a user