[ROCm][AITER] Skip AITER norm kernels when flattening to 2D would copy (#55991)
Signed-off-by: Zheng Gong <zgong@amd.com> Co-authored-by: Cursor <cursoragent@cursor.com> Co-authored-by: TJian <tunjian.tan@embeddedllm.com>
This commit is contained in:
co-authored by
Cursor
TJian
parent
f30a195bbb
commit
ab35354c21
@@ -0,0 +1,60 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Tests for the AITER norm support predicates.
|
||||
|
||||
The AITER norm wrappers flatten >2-D activations with ``Tensor.reshape``, which
|
||||
silently copies when the flattened shape is not expressible with the input's
|
||||
strides. The support predicates must reject those inputs so dispatch falls
|
||||
through to a provider that handles arbitrary strides.
|
||||
"""
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
from vllm._aiter_ops import is_aiter_found_and_supported
|
||||
from vllm.kernels.aiter_ops import flatten_to_2d_is_free
|
||||
|
||||
pytestmark = pytest.mark.skipif(
|
||||
not is_aiter_found_and_supported(),
|
||||
reason="Only test on ROCm with AITER installed and supported",
|
||||
)
|
||||
|
||||
|
||||
def _qkv_slice_by_head(num_tokens, num_q_heads, num_kv_heads, head_dim):
|
||||
"""q viewed per-head from a fused QKV projection, as Qwen3-style QK-norm does."""
|
||||
q_size, kv_size = num_q_heads * head_dim, num_kv_heads * head_dim
|
||||
qkv = torch.empty(num_tokens, q_size + 2 * kv_size)
|
||||
q = qkv.split([q_size, kv_size, kv_size], dim=-1)[0]
|
||||
return q.view(*q.shape[:-1], num_q_heads, head_dim)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"x",
|
||||
[
|
||||
torch.empty(8, 16),
|
||||
torch.empty(16),
|
||||
torch.empty(2, 4, 16),
|
||||
torch.empty(2, 1, 16),
|
||||
# A contiguous tensor stays flattenable after a leading-dim slice.
|
||||
torch.empty(8, 4, 16)[2:6],
|
||||
],
|
||||
)
|
||||
def test_flattenable(x):
|
||||
assert flatten_to_2d_is_free(x)
|
||||
assert x.reshape(-1, x.shape[-1]).data_ptr() == x.data_ptr()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"x",
|
||||
[
|
||||
_qkv_slice_by_head(8, 32, 8, 128),
|
||||
# Non-unit last-dim stride.
|
||||
torch.empty(4, 8, 32).transpose(-1, -2),
|
||||
# Leading dims cannot be merged: a slice along the middle dim.
|
||||
torch.empty(4, 8, 32)[:, :4],
|
||||
],
|
||||
)
|
||||
def test_not_flattenable(x):
|
||||
assert not flatten_to_2d_is_free(x)
|
||||
# reshape has to copy, which is exactly what the predicate guards against.
|
||||
assert x.reshape(-1, x.shape[-1]).data_ptr() != x.data_ptr()
|
||||
@@ -34,13 +34,38 @@ direct_register_aiter_op = functools.partial(
|
||||
AITER_SUPPORTED = is_aiter_found()
|
||||
"""Most kernels in this file are supported if AITER is installed."""
|
||||
|
||||
|
||||
def flatten_to_2d_is_free(x: Tensor) -> bool:
|
||||
"""Whether ``x.reshape(-1, x.shape[-1])`` is a view with unit-stride rows.
|
||||
|
||||
The AITER norm kernels only take dense 2D inputs, so the wrappers below
|
||||
flatten the leading dims with ``Tensor.reshape``, which silently falls back
|
||||
to ``contiguous()`` when the flattened shape is not expressible with the
|
||||
existing strides, adding a whole-tensor device copy that costs more than
|
||||
the kernel saves. A strided last dim flattens without a copy but not into a
|
||||
dense buffer, so it is rejected too.
|
||||
"""
|
||||
if x.dim() <= 2:
|
||||
return True
|
||||
if x.stride(-1) != 1:
|
||||
return False
|
||||
expected_stride = x.size(-1)
|
||||
for i in range(x.dim() - 2, -1, -1):
|
||||
if x.size(i) != 1 and x.stride(i) != expected_stride:
|
||||
return False
|
||||
expected_stride *= x.size(i)
|
||||
return True
|
||||
|
||||
|
||||
rms_no_var_16bit_only = (
|
||||
lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
and flatten_to_2d_is_free(x)
|
||||
)
|
||||
"""AITER rms_norm only supports float16 and bfloat16 acts, no var_size override,
|
||||
and requires weight dtype to match x dtype."""
|
||||
requires weight dtype to match x dtype, and requires flattening the activation
|
||||
to 2D to be free."""
|
||||
|
||||
|
||||
@ir.ops.rms_norm.register_impl(
|
||||
@@ -79,10 +104,13 @@ rms_add_no_var_16bit_only = (
|
||||
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
|
||||
and x.dtype in (torch.float16, torch.bfloat16)
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
and flatten_to_2d_is_free(x)
|
||||
and flatten_to_2d_is_free(x_residual)
|
||||
)
|
||||
"""
|
||||
AITER fused_add_rms_norm only supports 16-bit activations and no var_size override.
|
||||
Requires weight dtype to match x dtype.
|
||||
Requires weight dtype to match x dtype, and flattening both the activation and the
|
||||
residual to 2D to be free.
|
||||
"""
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user