[ROCm][CI] Fix ROCm LoRA Transformers fallback with full CUDA graphs (#41577)

Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Andreas Karatzas
2026-05-23 04:31:32 +00:00
committed by GitHub
parent 84e351555a
commit d28bdf9344
4 changed files with 88 additions and 19 deletions
+51
View File
@@ -141,6 +141,29 @@ def test_aiter_rejects_unsupported_dtypes():
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm vllm_c RMSNorm needs explicit ND input handling",
)
def test_vllm_c_rms_norm_accepts_nd_input():
torch.set_default_device(current_platform.device_type)
impl = ir.ops.rms_norm.impls["vllm_c"]
if not impl.supported:
pytest.skip("vllm_c impl not supported on this platform")
base = torch.randn(3, 8, 192, dtype=torch.float16)
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
assert not x.is_contiguous()
weight = torch.randn(16, dtype=torch.float16)
epsilon = 1e-5
output = impl.impl_fn(x, weight, epsilon)
ref_output = rms_norm_native(x, weight, epsilon)
assert output.shape == x.shape
assert_close(ir.ops.rms_norm, output, ref_output)
fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn
@@ -167,6 +190,34 @@ def test_fused_add_rms_norm_registration():
assert actual == expected
@pytest.mark.skipif(
not current_platform.is_rocm(),
reason="ROCm vllm_c fused_add_rms_norm needs explicit ND input handling",
)
def test_vllm_c_fused_add_rms_norm_accepts_nd_input():
torch.set_default_device(current_platform.device_type)
impl = ir.ops.fused_add_rms_norm.impls["vllm_c"]
if not impl.supported:
pytest.skip("vllm_c impl not supported on this platform")
base = torch.randn(3, 8, 192, dtype=torch.float16)
residual_base = torch.randn(3, 8, 192, dtype=torch.float16)
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
x_residual = residual_base.split(64, dim=-1)[0].view(3, 8, 4, 16)
assert not x.is_contiguous()
assert not x_residual.is_contiguous()
weight = torch.randn(16, dtype=torch.float16)
epsilon = 1e-5
output, residual = impl.impl_fn(x.clone(), x_residual.clone(), weight, epsilon)
ref_output, ref_residual = fused_add_rms_norm_native(x, x_residual, weight, epsilon)
assert output.shape == x.shape
assert residual.shape == x_residual.shape
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
+34 -6
View File
@@ -10,10 +10,11 @@ current_platform.import_kernels()
CUDA_ALIKE = current_platform.is_cuda_alike()
"""Most kernels in this file are supported on all CUDA-alike platforms."""
IS_ROCM = current_platform.is_rocm()
"""ROCm needs shape normalization before calling some vLLM C kernels."""
rms_no_var_size = (
lambda x, weight, epsilon, variance_size=None: variance_size is None
and (weight is None or weight.dtype == x.dtype)
rms_no_var_size = lambda x, weight, epsilon, variance_size=None: (
variance_size is None and (weight is None or weight.dtype == x.dtype)
)
"""vLLM kernel requires no variance_size override and matching input/weight dtype."""
@@ -28,14 +29,23 @@ def rms_norm(
# Kernel requires weight tensor, pass ones
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
assert variance_size is None
# ROCm's vLLM C RMSNorm kernel operates on contiguous 2D tensors.
# Higher-rank callers still normalize over the last dimension, so flatten
# all leading dims. reshape handles strided views from q/k/v splits.
if IS_ROCM and (x.dim() > 2 or not x.is_contiguous()):
original_shape = x.shape
x = x.reshape(-1, original_shape[-1])
output = torch.empty_like(x)
torch.ops._C.rms_norm(output, x, weight, epsilon)
return output.reshape(original_shape)
output = torch.empty(x.shape, device=x.device, dtype=x.dtype)
torch.ops._C.rms_norm(output, x, weight, epsilon)
return output
rms_add_no_var_size = (
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
and (weight is None or weight.dtype == x.dtype)
rms_add_no_var_size = lambda x, x_residual, weight, epsilon, variance_size=None: (
variance_size is None and (weight is None or weight.dtype == x.dtype)
)
"""vLLM Kernel does not support variance_size parameter and requires
matching input/weight dtype."""
@@ -59,5 +69,23 @@ def fused_add_rms_norm(
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
assert variance_size is None
if IS_ROCM and (not x.is_contiguous() or not x_residual.is_contiguous()):
output, residual = ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
x, x_residual, weight, epsilon
)
x.copy_(output)
x_residual.copy_(residual)
return x, x_residual
# ROCm's vLLM C RMSNorm kernel operates on contiguous 2D tensors.
# Higher-rank callers still normalize over the last dimension, so flatten
# all leading dims.
if IS_ROCM and x.dim() > 2:
original_shape = x.shape
x = x.view(-1, original_shape[-1])
x_residual = x_residual.view(-1, original_shape[-1])
torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon)
return x.view(original_shape), x_residual.view(original_shape)
torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon)
return x, x_residual
+1
View File
@@ -150,6 +150,7 @@ def rocm_unquantized_gemm_impl(
envs.VLLM_ROCM_USE_SKINNY_GEMM
and on_gfx950()
and x.dtype in [torch.float16, torch.bfloat16]
and x.dim() == 2
and (
10 <= n <= 128
and k % 8 == 0
@@ -16,7 +16,6 @@
# limitations under the License.
"""Transformers modeling backend base class."""
import sys
from collections.abc import Callable, Iterable
from itertools import chain
from operator import attrgetter
@@ -268,21 +267,11 @@ class Base(
dynamic_arg_dims,
)
@support_torch_compile(
support_torch_compile(
dynamic_arg_dims=dynamic_arg_dims,
enable_if=enable_if,
is_encoder=is_encoder,
)
class SupportTorchCompileWrapper(cls): ...
# Preserve __module__ so transformers v5's source-file checks
# (e.g. _can_set_experts_implementation) read the original
# model's module instead of this file.
SupportTorchCompileWrapper.__module__ = cls.__module__
# Patch the class in its module
module = sys.modules[cls.__module__]
setattr(module, cls.__name__, SupportTorchCompileWrapper)
)(cls)
def _decorate_for_torch_compile(self, **kwargs: dict):
"""