Weight sync refactor + move sparse nccl engine (#44353)

Signed-off-by: hao-aaron <ahao@anyscale.com>
Signed-off-by: haoaaron <ahao@anyscale.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Aaron Hao
2026-07-01 01:25:19 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.6
parent f651a8a9a4
commit 77a9c5ae28
32 changed files with 951 additions and 1044 deletions
@@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer(
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
device: torch.device | str = "cuda",
) -> None:
"""Consume packed tensors and unpack them into a list of tensors.
@@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer(
Both producer and consumer must use the same value.
num_buffers: Number of buffers for double/triple buffering.
Both producer and consumer must use the same value.
device: Device for the receive buffers. Must match the device the NCCL
communicator was created on (the worker's assigned device).
"""
target_packed_tensor_size = buffer_size_bytes
@@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer(
]
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
packed_tensors: list[torch.Tensor] = [
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
]
while True:
@@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer(
break
# Create a packed tensor and broadcast it
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda"
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model
@@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer(
packed_tensors[buffer_idx] = torch.empty(
packing_tensor_sizes[buffer_idx],
dtype=torch.uint8,
device="cuda",
device=device,
)
group.broadcast(packed_tensors[buffer_idx], src=src)
# Load the packed tensor into the model