[Perf] Low-latency CPU all-reduce for DP metadata syncs (#61767)

Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: hjjq <50634613+hjjq@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Nick Hill
2026-10-15 15:29:13 -07:00
committed by GitHub
co-authored by hjjq Claude Opus 5.5
parent 784d1fff0c
commit de7d70676d
7 changed files with 159 additions and 8 deletions
+64
View File
@@ -55,6 +55,70 @@ def ensure_divisibility(numerator, denominator):
)
# P2P tags [base, base + log2(group size) + 1] are reserved for cpu_all_reduce_.
_CPU_ALL_REDUCE_TAG_BASE = 27100
def cpu_all_reduce_(tensor: torch.Tensor, group: ProcessGroup) -> torch.Tensor:
"""Low-latency in-place SUM all-reduce of a small CPU tensor over a gloo group.
Recursive doubling within power-of-two blocks, then one concurrent exchange
between blocks. Results are bitwise identical across ranks. Works with
stateless (unregistered) process groups.
"""
world_size = group.size()
if world_size == 1:
return tensor
block_size = world_size & -world_size
num_blocks = world_size // block_size
if tensor.device.type != "cpu" or not tensor.is_contiguous():
torch.distributed.all_reduce(tensor, group=group)
return tensor
rank = group.rank()
tag = _CPU_ALL_REDUCE_TAG_BASE
# Recursive doubling within the block (peers stay in the aligned block).
if block_size > 1:
# This branch applies only to even world sizes.
scratch = torch.empty_like(tensor)
mask = 1
while mask < block_size:
peer = rank ^ mask
# Wait for both ops before touching either buffer.
recv_work = group.recv([scratch], peer, tag)
send_work = group.send([tensor], peer, tag)
recv_work.wait()
send_work.wait()
tensor.add_(scratch)
mask <<= 1
tag += 1
# Exchange block sums with the same lane in every other block at once.
if num_blocks > 1:
# This branch applies only when world_size is not a power of two.
lane, my_block = rank % block_size, rank // block_size
block_sums = [
tensor if b == my_block else torch.empty_like(tensor)
for b in range(num_blocks)
]
works = []
for b, buf in enumerate(block_sums):
if b != my_block:
works.append(group.recv([buf], b * block_size + lane, tag))
for b in range(num_blocks):
if b != my_block:
works.append(group.send([tensor], b * block_size + lane, tag))
for work in works:
work.wait()
# Sum in block order so every rank computes the identical result.
total = block_sums[0]
for buf in block_sums[1:]:
total.add_(buf)
tensor.copy_(total)
return tensor
def divide(numerator, denominator):
"""Ensure that numerator is divisible by the denominator and return
the division value."""