Weight sync refactor + move sparse nccl engine (#44353)
Signed-off-by: hao-aaron <ahao@anyscale.com> Signed-off-by: haoaaron <ahao@anyscale.com> Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.6
parent
f651a8a9a4
commit
77a9c5ae28
@@ -186,6 +186,7 @@ def packed_nccl_broadcast_consumer(
|
||||
post_unpack_func: Callable[[list[tuple[str, torch.Tensor]]], None],
|
||||
buffer_size_bytes: int = DEFAULT_PACKED_BUFFER_SIZE_BYTES,
|
||||
num_buffers: int = DEFAULT_PACKED_NUM_BUFFERS,
|
||||
device: torch.device | str = "cuda",
|
||||
) -> None:
|
||||
"""Consume packed tensors and unpack them into a list of tensors.
|
||||
|
||||
@@ -199,6 +200,8 @@ def packed_nccl_broadcast_consumer(
|
||||
Both producer and consumer must use the same value.
|
||||
num_buffers: Number of buffers for double/triple buffering.
|
||||
Both producer and consumer must use the same value.
|
||||
device: Device for the receive buffers. Must match the device the NCCL
|
||||
communicator was created on (the worker's assigned device).
|
||||
|
||||
"""
|
||||
target_packed_tensor_size = buffer_size_bytes
|
||||
@@ -211,7 +214,7 @@ def packed_nccl_broadcast_consumer(
|
||||
]
|
||||
packing_tensor_sizes: list[int] = [0 for _ in range(num_buffers)]
|
||||
packed_tensors: list[torch.Tensor] = [
|
||||
torch.empty(0, dtype=torch.uint8, device="cuda") for _ in range(num_buffers)
|
||||
torch.empty(0, dtype=torch.uint8, device=device) for _ in range(num_buffers)
|
||||
]
|
||||
|
||||
while True:
|
||||
@@ -234,7 +237,7 @@ def packed_nccl_broadcast_consumer(
|
||||
break
|
||||
# Create a packed tensor and broadcast it
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device="cuda"
|
||||
packing_tensor_sizes[buffer_idx], dtype=torch.uint8, device=device
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
@@ -259,7 +262,7 @@ def packed_nccl_broadcast_consumer(
|
||||
packed_tensors[buffer_idx] = torch.empty(
|
||||
packing_tensor_sizes[buffer_idx],
|
||||
dtype=torch.uint8,
|
||||
device="cuda",
|
||||
device=device,
|
||||
)
|
||||
group.broadcast(packed_tensors[buffer_idx], src=src)
|
||||
# Load the packed tensor into the model
|
||||
|
||||
Reference in New Issue
Block a user