From ab35354c21cc3c36439e79e18070f85cfafc3af2 Mon Sep 17 00:00:00 2001 From: Zheng Gong Date: Wed, 16 Sep 2026 15:26:56 +0800 Subject: [PATCH] [ROCm][AITER] Skip AITER norm kernels when flattening to 2D would copy (#55991) Signed-off-by: Zheng Gong Co-authored-by: Cursor Co-authored-by: TJian --- tests/kernels/ir/test_aiter_norm_dispatch.py | 60 ++++++++++++++++++++ vllm/kernels/aiter_ops.py | 32 ++++++++++- 2 files changed, 90 insertions(+), 2 deletions(-) create mode 100644 tests/kernels/ir/test_aiter_norm_dispatch.py diff --git a/tests/kernels/ir/test_aiter_norm_dispatch.py b/tests/kernels/ir/test_aiter_norm_dispatch.py new file mode 100644 index 0000000000..19cf5d3112 --- /dev/null +++ b/tests/kernels/ir/test_aiter_norm_dispatch.py @@ -0,0 +1,60 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Tests for the AITER norm support predicates. + +The AITER norm wrappers flatten >2-D activations with ``Tensor.reshape``, which +silently copies when the flattened shape is not expressible with the input's +strides. The support predicates must reject those inputs so dispatch falls +through to a provider that handles arbitrary strides. +""" + +import pytest +import torch + +from vllm._aiter_ops import is_aiter_found_and_supported +from vllm.kernels.aiter_ops import flatten_to_2d_is_free + +pytestmark = pytest.mark.skipif( + not is_aiter_found_and_supported(), + reason="Only test on ROCm with AITER installed and supported", +) + + +def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim): + """q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does.""" + q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim + qkv = torch.empty(num_tokens, q_size + 2 * kv_size) + q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0] + return q.view(*q.shape[:-1], num_q_heads, head_dim) + + +@pytest.mark.parametrize( + "x", + [ + torch.empty(8, 16), + torch.empty(16), + torch.empty(2, 4, 16), + torch.empty(2, 1, 16), + # A contiguous tensor stays flattenable after a leading-dim slice. + torch.empty(8, 4, 16)[2:6], + ], +) +def test_flattenable(x): + assert flatten_to_2d_is_free(x) + assert x.reshape(-1, x.shape[-1]).data_ptr() == x.data_ptr() + + +@pytest.mark.parametrize( + "x", + [ + _qkv_slice_by_head(8, 32, 8, 128), + # Non-unit last-dim stride. + torch.empty(4, 8, 32).transpose(-1, -2), + # Leading dims cannot be merged: a slice along the middle dim. + torch.empty(4, 8, 32)[:, :4], + ], +) +def test_not_flattenable(x): + assert not flatten_to_2d_is_free(x) + # reshape has to copy, which is exactly what the predicate guards against. + assert x.reshape(-1, x.shape[-1]).data_ptr() != x.data_ptr() diff --git a/vllm/kernels/aiter_ops.py b/vllm/kernels/aiter_ops.py index de8a53ae8b..402c9cac17 100644 --- a/vllm/kernels/aiter_ops.py +++ b/vllm/kernels/aiter_ops.py @@ -34,13 +34,38 @@ direct_register_aiter_op = functools.partial( AITER_SUPPORTED = is_aiter_found() """Most kernels in this file are supported if AITER is installed.""" + +def flatten_to_2d_is_free(x: Tensor) -> bool: + """Whether ``x.reshape(-1, x.shape[-1])`` is a view with unit-stride rows. + + The AITER norm kernels only take dense 2D inputs, so the wrappers below + flatten the leading dims with ``Tensor.reshape``, which silently falls back + to ``contiguous()`` when the flattened shape is not expressible with the + existing strides, adding a whole-tensor device copy that costs more than + the kernel saves. A strided last dim flattens without a copy but not into a + dense buffer, so it is rejected too. + """ + if x.dim() <= 2: + return True + if x.stride(-1) != 1: + return False + expected_stride = x.size(-1) + for i in range(x.dim() - 2, -1, -1): + if x.size(i) != 1 and x.stride(i) != expected_stride: + return False + expected_stride *= x.size(i) + return True + + rms_no_var_16bit_only = ( lambda x, weight, epsilon, variance_size=None: variance_size is None and x.dtype in (torch.float16, torch.bfloat16) and (weight is None or weight.dtype == x.dtype) + and flatten_to_2d_is_free(x) ) """AITER rms_norm only supports float16 and bfloat16 acts, no var_size override, -and requires weight dtype to match x dtype.""" +requires weight dtype to match x dtype, and requires flattening the activation +to 2D to be free.""" @ir.ops.rms_norm.register_impl( @@ -79,10 +104,13 @@ rms_add_no_var_16bit_only = ( lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None and x.dtype in (torch.float16, torch.bfloat16) and (weight is None or weight.dtype == x.dtype) + and flatten_to_2d_is_free(x) + and flatten_to_2d_is_free(x_residual) ) """ AITER fused_add_rms_norm only supports 16-bit activations and no var_size override. -Requires weight dtype to match x dtype. +Requires weight dtype to match x dtype, and flattening both the activation and the +residual to 2D to be free. """