From 9c78c99995b70726f9ea929ff2e535d6303383d6 Mon Sep 17 00:00:00 2001 From: Vadim Gimpelson <156319763+vadiklyutiy@users.noreply.github.com> Date: Wed, 20 May 2026 19:50:24 +0400 Subject: [PATCH] [MISC] Fix symm_mem cap-equal gate; log AR backend selection (#42993) Signed-off-by: Vadim Gimpelson --- .../device_communicators/cuda_communicator.py | 67 +++++++++++++++++++ .../device_communicators/symm_mem.py | 2 +- 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index e46d57cf92..ee81bc20f3 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -7,6 +7,7 @@ from torch.distributed import ProcessGroup import vllm.envs as envs from vllm.distributed.device_communicators.all_reduce_utils import ( + NCCL_SYMM_MEM_ALL_REDUCE_CONFIG, should_nccl_symm_mem_allreduce, ) from vllm.distributed.device_communicators.pynccl import register_nccl_symmetric_ops @@ -114,6 +115,9 @@ class CudaCommunicator(DeviceCommunicatorBase): # currently be an MI300 series. self.qr_comm = QuickAllReduce(group=self.cpu_group, device=self.device) + if self.world_size > 1: + self._log_all_reduce_backend_selection() + if self.use_all2all: if self.all2all_backend in ("naive", "allgather_reducescatter"): from .all2all import AgRsAll2AllManager @@ -171,6 +175,69 @@ class CudaCommunicator(DeviceCommunicatorBase): scope="global", ) + def _log_all_reduce_backend_selection(self) -> None: + """Log the all-reduce backends that are active for this group. + + The dispatch chain in ``all_reduce`` tries backends in this order and + falls through to the next one if the current backend rejects the + input (size/dtype gates) or is disabled. The list of "enabled" + backends below is the subset of potential backends that may be + chosen at dispatch time for this group; the actual per-call choice + depends on the input tensor. + """ + all_potential_ar_backends = [ + "NCCL_SYMM_MEM", + "QUICK_REDUCE", + "FLASHINFER", + "CUSTOM", + "SYMM_MEM", + "PYNCCL", + ] + enabled_ar_backends: list[str] = [] + # Mirror the static preconditions of `should_nccl_symm_mem_allreduce`: + # VLLM_BATCH_INVARIANT off, NCCL symm mem enabled, world_size meets + # min_world_size, and world_size either has a tuned entry in + # `custom_ar_preferred_ranges` or is greater than + # `always_use_above_world_size`. World sizes that fail the latter (e.g. + # 5/6/7 with the default config) never dispatch NCCL symm mem + # regardless of input. The per-tensor-size check inside the function + # stays as a runtime decision. + nccl_symm_ws_ok = self.world_size >= NCCL_SYMM_MEM_ALL_REDUCE_CONFIG[ + "min_world_size" + ] and ( + self.world_size + in NCCL_SYMM_MEM_ALL_REDUCE_CONFIG["custom_ar_preferred_ranges"] + or self.world_size + > NCCL_SYMM_MEM_ALL_REDUCE_CONFIG["always_use_above_world_size"] + ) + if ( + self.pynccl_comm is not None + and not self.pynccl_comm.disabled + and is_symmetric_memory_enabled() + and not envs.VLLM_BATCH_INVARIANT + and nccl_symm_ws_ok + ): + enabled_ar_backends.append("NCCL_SYMM_MEM") + if self.qr_comm is not None and not self.qr_comm.disabled: + enabled_ar_backends.append("QUICK_REDUCE") + if self.fi_ar_comm is not None and not self.fi_ar_comm.disabled: + enabled_ar_backends.append("FLASHINFER") + if self.ca_comm is not None and not self.ca_comm.disabled: + enabled_ar_backends.append("CUSTOM") + if self.symm_mem_comm is not None and not self.symm_mem_comm.disabled: + enabled_ar_backends.append("SYMM_MEM") + if self.pynccl_comm is not None and not self.pynccl_comm.disabled: + enabled_ar_backends.append("PYNCCL") + + logger.info_once( + "Using %s all-reduce backends (in dispatch order) for group " + "'%s' out of potential backends: %s.", + "[" + ", ".join(f"'{b}'" for b in enabled_ar_backends) + "]", + self.unique_name or "", + "[" + ", ".join(f"'{b}'" for b in all_potential_ar_backends) + "]", + scope="global", + ) + def all_reduce(self, input_): # since currently we perform copy input -> symm_input -> out-of-place AR # return symm_output, we don't need to check if input is symmetric diff --git a/vllm/distributed/device_communicators/symm_mem.py b/vllm/distributed/device_communicators/symm_mem.py index 3d964c640d..8c174602c3 100644 --- a/vllm/distributed/device_communicators/symm_mem.py +++ b/vllm/distributed/device_communicators/symm_mem.py @@ -121,7 +121,7 @@ class SymmMemCommunicator: inp_size = inp.numel() * inp.element_size() if inp_size % 4 != 0: return False - return inp_size < self.max_size + return inp_size <= self.max_size def all_reduce( self, inp: torch.Tensor, *, out: torch.Tensor | None = None