[ROCm][CI] Fix ROCm LoRA Transformers fallback with full CUDA graphs (#41577)
Signed-off-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
@@ -141,6 +141,29 @@ def test_aiter_rejects_unsupported_dtypes():
|
||||
assert not impl.supports_args(*args), f"aiter should reject dtype={dtype}"
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm vllm_c RMSNorm needs explicit ND input handling",
|
||||
)
|
||||
def test_vllm_c_rms_norm_accepts_nd_input():
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
impl = ir.ops.rms_norm.impls["vllm_c"]
|
||||
if not impl.supported:
|
||||
pytest.skip("vllm_c impl not supported on this platform")
|
||||
|
||||
base = torch.randn(3, 8, 192, dtype=torch.float16)
|
||||
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
|
||||
assert not x.is_contiguous()
|
||||
weight = torch.randn(16, dtype=torch.float16)
|
||||
epsilon = 1e-5
|
||||
|
||||
output = impl.impl_fn(x, weight, epsilon)
|
||||
ref_output = rms_norm_native(x, weight, epsilon)
|
||||
|
||||
assert output.shape == x.shape
|
||||
assert_close(ir.ops.rms_norm, output, ref_output)
|
||||
|
||||
|
||||
fused_add_rms_norm_native = ir.ops.fused_add_rms_norm.impls["native"].impl_fn
|
||||
|
||||
|
||||
@@ -167,6 +190,34 @@ def test_fused_add_rms_norm_registration():
|
||||
assert actual == expected
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not current_platform.is_rocm(),
|
||||
reason="ROCm vllm_c fused_add_rms_norm needs explicit ND input handling",
|
||||
)
|
||||
def test_vllm_c_fused_add_rms_norm_accepts_nd_input():
|
||||
torch.set_default_device(current_platform.device_type)
|
||||
impl = ir.ops.fused_add_rms_norm.impls["vllm_c"]
|
||||
if not impl.supported:
|
||||
pytest.skip("vllm_c impl not supported on this platform")
|
||||
|
||||
base = torch.randn(3, 8, 192, dtype=torch.float16)
|
||||
residual_base = torch.randn(3, 8, 192, dtype=torch.float16)
|
||||
x = base.split(64, dim=-1)[0].view(3, 8, 4, 16)
|
||||
x_residual = residual_base.split(64, dim=-1)[0].view(3, 8, 4, 16)
|
||||
assert not x.is_contiguous()
|
||||
assert not x_residual.is_contiguous()
|
||||
weight = torch.randn(16, dtype=torch.float16)
|
||||
epsilon = 1e-5
|
||||
|
||||
output, residual = impl.impl_fn(x.clone(), x_residual.clone(), weight, epsilon)
|
||||
ref_output, ref_residual = fused_add_rms_norm_native(x, x_residual, weight, epsilon)
|
||||
|
||||
assert output.shape == x.shape
|
||||
assert residual.shape == x_residual.shape
|
||||
assert_close(ir.ops.fused_add_rms_norm, output, ref_output)
|
||||
assert_close(ir.ops.fused_add_rms_norm, residual, ref_residual)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", [torch.float16, torch.bfloat16, torch.float32])
|
||||
@pytest.mark.parametrize("n_tokens", NUM_TOKENS)
|
||||
@pytest.mark.parametrize("hidden_size", COMMON_HIDDEN_SIZES)
|
||||
|
||||
+34
-6
@@ -10,10 +10,11 @@ current_platform.import_kernels()
|
||||
|
||||
CUDA_ALIKE = current_platform.is_cuda_alike()
|
||||
"""Most kernels in this file are supported on all CUDA-alike platforms."""
|
||||
IS_ROCM = current_platform.is_rocm()
|
||||
"""ROCm needs shape normalization before calling some vLLM C kernels."""
|
||||
|
||||
rms_no_var_size = (
|
||||
lambda x, weight, epsilon, variance_size=None: variance_size is None
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
rms_no_var_size = lambda x, weight, epsilon, variance_size=None: (
|
||||
variance_size is None and (weight is None or weight.dtype == x.dtype)
|
||||
)
|
||||
"""vLLM kernel requires no variance_size override and matching input/weight dtype."""
|
||||
|
||||
@@ -28,14 +29,23 @@ def rms_norm(
|
||||
# Kernel requires weight tensor, pass ones
|
||||
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
|
||||
assert variance_size is None
|
||||
# ROCm's vLLM C RMSNorm kernel operates on contiguous 2D tensors.
|
||||
# Higher-rank callers still normalize over the last dimension, so flatten
|
||||
# all leading dims. reshape handles strided views from q/k/v splits.
|
||||
if IS_ROCM and (x.dim() > 2 or not x.is_contiguous()):
|
||||
original_shape = x.shape
|
||||
x = x.reshape(-1, original_shape[-1])
|
||||
output = torch.empty_like(x)
|
||||
torch.ops._C.rms_norm(output, x, weight, epsilon)
|
||||
return output.reshape(original_shape)
|
||||
|
||||
output = torch.empty(x.shape, device=x.device, dtype=x.dtype)
|
||||
torch.ops._C.rms_norm(output, x, weight, epsilon)
|
||||
return output
|
||||
|
||||
|
||||
rms_add_no_var_size = (
|
||||
lambda x, x_residual, weight, epsilon, variance_size=None: variance_size is None
|
||||
and (weight is None or weight.dtype == x.dtype)
|
||||
rms_add_no_var_size = lambda x, x_residual, weight, epsilon, variance_size=None: (
|
||||
variance_size is None and (weight is None or weight.dtype == x.dtype)
|
||||
)
|
||||
"""vLLM Kernel does not support variance_size parameter and requires
|
||||
matching input/weight dtype."""
|
||||
@@ -59,5 +69,23 @@ def fused_add_rms_norm(
|
||||
weight = torch.ones(x.shape[-1], device=x.device, dtype=x.dtype)
|
||||
|
||||
assert variance_size is None
|
||||
if IS_ROCM and (not x.is_contiguous() or not x_residual.is_contiguous()):
|
||||
output, residual = ir.ops.fused_add_rms_norm.impls["native"].impl_fn(
|
||||
x, x_residual, weight, epsilon
|
||||
)
|
||||
x.copy_(output)
|
||||
x_residual.copy_(residual)
|
||||
return x, x_residual
|
||||
|
||||
# ROCm's vLLM C RMSNorm kernel operates on contiguous 2D tensors.
|
||||
# Higher-rank callers still normalize over the last dimension, so flatten
|
||||
# all leading dims.
|
||||
if IS_ROCM and x.dim() > 2:
|
||||
original_shape = x.shape
|
||||
x = x.view(-1, original_shape[-1])
|
||||
x_residual = x_residual.view(-1, original_shape[-1])
|
||||
torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon)
|
||||
return x.view(original_shape), x_residual.view(original_shape)
|
||||
|
||||
torch.ops._C.fused_add_rms_norm(x, x_residual, weight, epsilon)
|
||||
return x, x_residual
|
||||
|
||||
@@ -150,6 +150,7 @@ def rocm_unquantized_gemm_impl(
|
||||
envs.VLLM_ROCM_USE_SKINNY_GEMM
|
||||
and on_gfx950()
|
||||
and x.dtype in [torch.float16, torch.bfloat16]
|
||||
and x.dim() == 2
|
||||
and (
|
||||
10 <= n <= 128
|
||||
and k % 8 == 0
|
||||
|
||||
@@ -16,7 +16,6 @@
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend base class."""
|
||||
|
||||
import sys
|
||||
from collections.abc import Callable, Iterable
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
@@ -268,21 +267,11 @@ class Base(
|
||||
dynamic_arg_dims,
|
||||
)
|
||||
|
||||
@support_torch_compile(
|
||||
support_torch_compile(
|
||||
dynamic_arg_dims=dynamic_arg_dims,
|
||||
enable_if=enable_if,
|
||||
is_encoder=is_encoder,
|
||||
)
|
||||
class SupportTorchCompileWrapper(cls): ...
|
||||
|
||||
# Preserve __module__ so transformers v5's source-file checks
|
||||
# (e.g. _can_set_experts_implementation) read the original
|
||||
# model's module instead of this file.
|
||||
SupportTorchCompileWrapper.__module__ = cls.__module__
|
||||
|
||||
# Patch the class in its module
|
||||
module = sys.modules[cls.__module__]
|
||||
setattr(module, cls.__name__, SupportTorchCompileWrapper)
|
||||
)(cls)
|
||||
|
||||
def _decorate_for_torch_compile(self, **kwargs: dict):
|
||||
"""
|
||||
|
||||
Reference in New Issue
Block a user