From f944491db914880dc5f12c0b53c1aac113014a4b Mon Sep 17 00:00:00 2001 From: Guanxin Li <38149783+guanxingithub@users.noreply.github.com> Date: Tue, 22 Sep 2026 11:44:00 -0700 Subject: [PATCH] [Bugfix] batch_invariant: keep non-AllReduce collectives enabled on NCCL >= 2.31 (#58179) Signed-off-by: Guanxin Li <38149783+guanxingithub@users.noreply.github.com> Co-authored-by: Flora Feng <4florafeng@gmail.com> --- vllm/model_executor/determinism/batch_invariant.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/vllm/model_executor/determinism/batch_invariant.py b/vllm/model_executor/determinism/batch_invariant.py index 08035d0e38..308e8c4f42 100644 --- a/vllm/model_executor/determinism/batch_invariant.py +++ b/vllm/model_executor/determinism/batch_invariant.py @@ -1150,7 +1150,12 @@ def override_envs_for_invariance(): os.environ["NCCL_MIN_NCHANNELS"] = "1" os.environ["NCCL_MAX_NCHANNELS"] = "1" os.environ["NCCL_PROTO"] = "Simple" - os.environ["NCCL_ALGO"] = "allreduce:tree" + # NCCL >= 2.31 zero-fills the algorithm table of every collective when + # NCCL_ALGO is set and re-enables only the named ones; together with the + # NCCL_PROTO above, collectives not named here end up with no algorithm + # and fail with ncclInvalidUsage. Re-enable Ring and Tree for all + # collectives, then pin AllReduce to Tree for determinism. + os.environ["NCCL_ALGO"] = "ring,tree;allreduce:tree" os.environ["NCCL_NTHREADS"] = "1" os.environ["NCCL_SOCKET_NTHREADS"] = "1"