[Bugfix] batch_invariant: keep non-AllReduce collectives enabled on NCCL >= 2.31 (#58179)

Signed-off-by: Guanxin Li <38149783+guanxingithub@users.noreply.github.com>
Co-authored-by: Flora Feng <4florafeng@gmail.com>
This commit is contained in:
Guanxin Li
2026-09-22 14:44:00 -04:00
committed by GitHub
co-authored by Flora Feng
parent c024cfdb02
commit f944491db9
@@ -1150,7 +1150,12 @@ def override_envs_for_invariance():
os.environ["NCCL_MIN_NCHANNELS"] = "1"
os.environ["NCCL_MAX_NCHANNELS"] = "1"
os.environ["NCCL_PROTO"] = "Simple"
os.environ["NCCL_ALGO"] = "allreduce:tree"
# NCCL >= 2.31 zero-fills the algorithm table of every collective when
# NCCL_ALGO is set and re-enables only the named ones; together with the
# NCCL_PROTO above, collectives not named here end up with no algorithm
# and fail with ncclInvalidUsage. Re-enable Ring and Tree for all
# collectives, then pin AllReduce to Tree for determinism.
os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree"
os.environ["NCCL_NTHREADS"] = "1"
os.environ["NCCL_SOCKET_NTHREADS"] = "1"