[Perf] Low-latency CPU all-reduce for DP metadata syncs (#61767)
Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: hjjq <50634613+hjjq@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
hjjq
Claude Opus 5.5
parent
784d1fff0c
commit
de7d70676d
@@ -55,6 +55,70 @@ def ensure_divisibility(numerator, denominator):
|
||||
)
|
||||
|
||||
|
||||
# P2P tags [base, base + log2(group size) + 1] are reserved for cpu_all_reduce_.
|
||||
_CPU_ALL_REDUCE_TAG_BASE = 27100
|
||||
|
||||
|
||||
def cpu_all_reduce_(tensor: torch.Tensor, group: ProcessGroup) -> torch.Tensor:
|
||||
"""Low-latency in-place SUM all-reduce of a small CPU tensor over a gloo group.
|
||||
|
||||
Recursive doubling within power-of-two blocks, then one concurrent exchange
|
||||
between blocks. Results are bitwise identical across ranks. Works with
|
||||
stateless (unregistered) process groups.
|
||||
"""
|
||||
world_size = group.size()
|
||||
if world_size == 1:
|
||||
return tensor
|
||||
block_size = world_size & -world_size
|
||||
num_blocks = world_size // block_size
|
||||
if tensor.device.type != "cpu" or not tensor.is_contiguous():
|
||||
torch.distributed.all_reduce(tensor, group=group)
|
||||
return tensor
|
||||
|
||||
rank = group.rank()
|
||||
tag = _CPU_ALL_REDUCE_TAG_BASE
|
||||
|
||||
# Recursive doubling within the block (peers stay in the aligned block).
|
||||
if block_size > 1:
|
||||
# This branch applies only to even world sizes.
|
||||
scratch = torch.empty_like(tensor)
|
||||
mask = 1
|
||||
while mask < block_size:
|
||||
peer = rank ^ mask
|
||||
# Wait for both ops before touching either buffer.
|
||||
recv_work = group.recv([scratch], peer, tag)
|
||||
send_work = group.send([tensor], peer, tag)
|
||||
recv_work.wait()
|
||||
send_work.wait()
|
||||
tensor.add_(scratch)
|
||||
mask <<= 1
|
||||
tag += 1
|
||||
|
||||
# Exchange block sums with the same lane in every other block at once.
|
||||
if num_blocks > 1:
|
||||
# This branch applies only when world_size is not a power of two.
|
||||
lane, my_block = rank % block_size, rank // block_size
|
||||
block_sums = [
|
||||
tensor if b == my_block else torch.empty_like(tensor)
|
||||
for b in range(num_blocks)
|
||||
]
|
||||
works = []
|
||||
for b, buf in enumerate(block_sums):
|
||||
if b != my_block:
|
||||
works.append(group.recv([buf], b * block_size + lane, tag))
|
||||
for b in range(num_blocks):
|
||||
if b != my_block:
|
||||
works.append(group.send([tensor], b * block_size + lane, tag))
|
||||
for work in works:
|
||||
work.wait()
|
||||
# Sum in block order so every rank computes the identical result.
|
||||
total = block_sums[0]
|
||||
for buf in block_sums[1:]:
|
||||
total.add_(buf)
|
||||
tensor.copy_(total)
|
||||
return tensor
|
||||
|
||||
|
||||
def divide(numerator, denominator):
|
||||
"""Ensure that numerator is divisible by the denominator and return
|
||||
the division value."""
|
||||
|
||||
Reference in New Issue
Block a user