diff --git a/tests/kernels/ir/test_layernorm.py b/tests/kernels/ir/test_layernorm.py index e9661f5202..6fc9d7543d 100644 --- a/tests/kernels/ir/test_layernorm.py +++ b/tests/kernels/ir/test_layernorm.py @@ -141,6 +141,29 @@ def test_aiter_rejects_unsupported_dtypes(): assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}" +@pytest.mark.skipif( + not current_platform.is_rocm(), + reason="ROCm vllm_c RMSNorm needs explicit ND input handling", +) +def test_vllm_c_rms_norm_accepts_nd_input(): + torch.set_default_device(current_platform.device_type) + impl = ir.ops.rms_norm.impls["vllm_c"] + if not impl.supported: + pytest.skip("vllm_c impl not supported on this platform") + + base = torch.randn(3, 8, 192, dtype=torch.float16) + x = base.split(64, dim=-1)[0].view(3, 8, 4, 16) + assert not x.is_contiguous() + weight = torch.randn(16, dtype=torch.float16) + epsilon = 1e-5 + + output = impl.impl_fn(x, weight, epsilon) + ref_output = rms_norm_native(x, weight, epsilon) + + assert output.shape == x.shape + assert_close(ir.ops.rms_norm, output, ref_output) + + fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn @@ -167,6 +190,34 @@ def test_fused_add_rms_norm_registration(): assert actual == expected +@pytest.mark.skipif( + not current_platform.is_rocm(), + reason="ROCm vllm_c fused_add_rms_norm needs explicit ND input handling", +) +def test_vllm_c_fused_add_rms_norm_accepts_nd_input(): + torch.set_default_device(current_platform.device_type) + impl = ir.ops.fused_add_rms_norm.impls["vllm_c"] + if not impl.supported: + pytest.skip("vllm_c impl not supported on this platform") + + base = torch.randn(3, 8, 192, dtype=torch.float16) + residual_base = torch.randn(3, 8, 192, dtype=torch.float16) + x = base.split(64, dim=-1)[0].view(3, 8, 4, 16) + x_residual = residual_base.split(64, dim=-1)[0].view(3, 8, 4, 16) + assert not x.is_contiguous() + assert not x_residual.is_contiguous() + weight = torch.randn(16, dtype=torch.float16) + epsilon = 1e-5 + + output, residual = impl.impl_fn(x.clone(), x_residual.clone(), weight, epsilon) + ref_output, ref_residual = fused_add_rms_norm_native(x, x_residual, weight, epsilon) + + assert output.shape == x.shape + assert residual.shape == x_residual.shape + assert_close(ir.ops.fused_add_rms_norm, output, ref_output) + assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual) + + @pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32]) @pytest.mark.parametrize("n_tokens", NUM_TOKENS) @pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES) diff --git a/vllm/kernels/vllm_c.py b/vllm/kernels/vllm_c.py index 5c602c3984..3b194b2ab9 100644 --- a/vllm/kernels/vllm_c.py +++ b/vllm/kernels/vllm_c.py @@ -10,10 +10,11 @@ current_platform.import_kernels() CUDA_ALIKE = current_platform.is_cuda_alike() """Most kernels in this file are supported on all CUDA-alike platforms.""" +IS_ROCM = current_platform.is_rocm() +"""ROCm needs shape normalization before calling some vLLM C kernels.""" -rms_no_var_size = ( - lambda x, weight, epsilon, variance_size=None: variance_size is None - and (weight is None or weight.dtype == x.dtype) +rms_no_var_size = lambda x, weight, epsilon, variance_size=None: ( + variance_size is None and (weight is None or weight.dtype == x.dtype) ) """vLLM kernel requires no variance_size override and matching input/weight dtype.""" @@ -28,14 +29,23 @@ def rms_norm( # Kernel requires weight tensor, pass ones weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype) assert variance_size is None + # ROCm's vLLM C RMSNorm kernel operates on contiguous 2D tensors. + # Higher-rank callers still normalize over the last dimension, so flatten + # all leading dims. reshape handles strided views from q/k/v splits. + if IS_ROCM and (x.dim() > 2 or not x.is_contiguous()): + original_shape = x.shape + x = x.reshape(-1, original_shape[-1]) + output = torch.empty_like(x) + torch.ops._C.rms_norm(output, x, weight, epsilon) + return output.reshape(original_shape) + output = torch.empty(x.shape, device=x.device, dtype=x.dtype) torch.ops._C.rms_norm(output, x, weight, epsilon) return output -rms_add_no_var_size = ( - lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None - and (weight is None or weight.dtype == x.dtype) +rms_add_no_var_size = lambda x, x_residual, weight, epsilon, variance_size=None: ( + variance_size is None and (weight is None or weight.dtype == x.dtype) ) """vLLM Kernel does not support variance_size parameter and requires matching input/weight dtype.""" @@ -59,5 +69,23 @@ def fused_add_rms_norm( weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype) assert variance_size is None + if IS_ROCM and (not x.is_contiguous() or not x_residual.is_contiguous()): + output, residual = ir.ops.fused_add_rms_norm.impls["native"].impl_fn( + x, x_residual, weight, epsilon + ) + x.copy_(output) + x_residual.copy_(residual) + return x, x_residual + + # ROCm's vLLM C RMSNorm kernel operates on contiguous 2D tensors. + # Higher-rank callers still normalize over the last dimension, so flatten + # all leading dims. + if IS_ROCM and x.dim() > 2: + original_shape = x.shape + x = x.view(-1, original_shape[-1]) + x_residual = x_residual.view(-1, original_shape[-1]) + torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon) + return x.view(original_shape), x_residual.view(original_shape) + torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon) return x, x_residual diff --git a/vllm/model_executor/layers/utils.py b/vllm/model_executor/layers/utils.py index 8bab7055e4..dbc05273f3 100644 --- a/vllm/model_executor/layers/utils.py +++ b/vllm/model_executor/layers/utils.py @@ -150,6 +150,7 @@ def rocm_unquantized_gemm_impl( envs.VLLM_ROCM_USE_SKINNY_GEMM and on_gfx950() and x.dtype in [torch.float16, torch.bfloat16] + and x.dim() == 2 and ( 10 <= n <= 128 and k % 8 == 0 diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index a3e4b844b8..35897ce7db 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -16,7 +16,6 @@ # limitations under the License. """Transformers modeling backend base class.""" -import sys from collections.abc import Callable, Iterable from itertools import chain from operator import attrgetter @@ -268,21 +267,11 @@ class Base( dynamic_arg_dims, ) - @support_torch_compile( + support_torch_compile( dynamic_arg_dims=dynamic_arg_dims, enable_if=enable_if, is_encoder=is_encoder, - ) - class SupportTorchCompileWrapper(cls): ... - - # Preserve __module__ so transformers v5's source-file checks - # (e.g. _can_set_experts_implementation) read the original - # model's module instead of this file. - SupportTorchCompileWrapper.__module__ = cls.__module__ - - # Patch the class in its module - module = sys.modules[cls.__module__] - setattr(module, cls.__name__, SupportTorchCompileWrapper) + )(cls) def _decorate_for_torch_compile(self, **kwargs: dict): """