diff --git a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py index 4de5c6cf7a..9f6d4e5a75 100644 --- a/vllm/compilation/passes/fusion/allreduce_rms_fusion.py +++ b/vllm/compilation/passes/fusion/allreduce_rms_fusion.py @@ -155,6 +155,13 @@ if flashinfer_comm is not None: scale_factor: torch.Tensor | None = None, weight_bias: float = 0.0, ) -> None: + # handle transformers backend passing outer batch dim. + if allreduce_in.dim() != 2: + hidden = allreduce_in.shape[-1] + allreduce_in = allreduce_in.view(-1, hidden) + residual = residual.view(-1, hidden) + if norm_out is not None: + norm_out = norm_out.view(-1, hidden) num_tokens, hidden_size = allreduce_in.shape element_size = allreduce_in.element_size() current_tensor_size = num_tokens * hidden_size * element_size