diff --git a/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py b/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py index e48d9c0135..5094662f74 100644 --- a/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py +++ b/vllm/model_executor/layers/minimax_rms_norm/rms_norm_tp.py @@ -6,6 +6,7 @@ from functools import partial import torch from torch import nn +from vllm._aiter_ops import rocm_aiter_ops from vllm.distributed.communication_op import tensor_model_parallel_all_reduce from vllm.distributed.parallel_state import ( get_tensor_model_parallel_rank, @@ -235,6 +236,17 @@ def _minimax_qk_norm_fusion( tp_world, eps, ) + # ROCm AITER fused QK-norm + AllReduce; skips unaligned shapes. + if tp_world > 1 and num_tokens <= MINIMAX_QK_NORM_MAX_TOKEN_NUM: + pack_size = 16 // qkv.element_size() + warp_work_size = 32 * pack_size + if q_size % warp_work_size == 0 and kv_size % warp_work_size == 0: + aiter_ar = rocm_aiter_ops.get_aiter_allreduce() + if aiter_ar is not None: + q_out, k_out, _ = aiter_ar.aiter_ca.custom_fused_qknorm_ar( + qkv, q_weight, k_weight, eps + ) + return q_out, k_out return _minimax_qk_norm_tp_fallback( qkv, q_weight, k_weight, q_size, kv_size, tp_rank, tp_world, eps )