From 775c1589ea9c7df49a866f900341e3645c5bd60c Mon Sep 17 00:00:00 2001 From: Fangzhou Ai <31551580+Fangzhou-Ai@users.noreply.github.com> Date: Sun, 12 Jul 2026 23:53:53 -0400 Subject: [PATCH] [Bugfix][ROCm] Keep TP all_gather on base-class collective (#48446) Signed-off-by: fai Co-authored-by: Cursor --- .../distributed/device_communicators/cuda_communicator.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/vllm/distributed/device_communicators/cuda_communicator.py b/vllm/distributed/device_communicators/cuda_communicator.py index a196a54f50..b613819477 100644 --- a/vllm/distributed/device_communicators/cuda_communicator.py +++ b/vllm/distributed/device_communicators/cuda_communicator.py @@ -352,6 +352,14 @@ class CudaCommunicator(DeviceCommunicatorBase): if pynccl_comm is None or pynccl_comm.disabled: return super().all_gather(input_, dim) + # On ROCm, the base-class all_gather (all_gather_into_tensor) is faster + # than the manual pynccl + torch.empty + movedim + reshape path below, + # which adds a per-call output allocation and (for dim != 0) an extra + # copy on every step. This is on the hot path for TP forward passes, so + # keep ROCm on the base-class collective to avoid a decode regression. + if current_platform.is_rocm(): + return super().all_gather(input_, dim) + input_size = input_.size() output_size = (input_size[0] * self.world_size,) + input_size[1:] output_tensor = torch.empty(