From ebf3a6d70521214d01f23baca7b0b4f92944abab Mon Sep 17 00:00:00 2001 From: Thomas Parnell Date: Tue, 16 Jun 2026 10:34:27 +0200 Subject: [PATCH] [Bugfix] Fix trtllm fused allreduce+rms_norm for transformers backend (#45307) Signed-off-by: Thomas Parnell --- vllm/compilation/passes/fusion/allreduce_rms_fusion.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 4de5c6cf7a..9f6d4e5a75 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -155,6 +155,13 @@ if flashinfer_comm is not None: scale_factor: torch.Tensor | None = None, weight_bias: float = 0.0, ) -> None: + # handle transformers backend passing outer batch dim. + if allreduce_in.dim() != 2: + hidden = allreduce_in.shape[-1] + allreduce_in = allreduce_in.view(-1, hidden) + residual = residual.view(-1, hidden) + if norm_out is not None: + norm_out = norm_out.view(-1, hidden) num_tokens, hidden_size = allreduce_in.shape element_size = allreduce_in.element_size() current_tensor_size = num_tokens * hidden_size * element_size