[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+3 -1
View File
@@ -578,7 +578,9 @@ class CudaPlatformBase(Platform):
if envs.VLLM_USE_OINK_OPS:
rms_norm = ["oink"] + default
return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm)
return IrOpPriorityConfig.with_default(
default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm
)
# NVML utils
+6 -16
View File
@@ -693,21 +693,11 @@ class RocmPlatform(Platform):
@classmethod
def apply_config_platform_defaults(cls, vllm_config: "VllmConfig") -> None:
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.compilation import CUDAGraphMode
compilation_config = vllm_config.compilation_config
is_eager_execution = compilation_config.cudagraph_mode == CUDAGraphMode.NONE
use_aiter_fused_moe = rocm_aiter_ops.is_fused_moe_enabled()
use_aiter_rms_norm = rocm_aiter_ops.is_rmsnorm_enabled()
use_aiter_fp8_linear = rocm_aiter_ops.is_linear_fp8_enabled()
use_aiter_fused_se = rocm_aiter_ops.is_fusion_moe_shared_experts_enabled()
# Aiter rms norm perform best when CUDA Graph capture is enabled.
if (
use_aiter_rms_norm
and not is_eager_execution
and "-rms_norm" not in compilation_config.custom_ops
):
compilation_config.custom_ops.append("+rms_norm")
if use_aiter_fp8_linear and "-quant_fp8" not in compilation_config.custom_ops:
compilation_config.custom_ops.append("+quant_fp8")
@@ -939,7 +929,7 @@ class RocmPlatform(Platform):
def get_default_ir_op_priority(
cls, vllm_config: "VllmConfig"
) -> "IrOpPriorityConfig":
from vllm.config.compilation import CompilationMode
from vllm.config.compilation import CompilationMode, CUDAGraphMode
from vllm.config.kernel import IrOpPriorityConfig
# Native used by default when compiling,
@@ -949,12 +939,10 @@ class RocmPlatform(Platform):
using_inductor = cc.backend == "inductor" and cc.mode != CompilationMode.NONE
default = ["native"] if using_inductor else ["vllm_c", "native"]
# This (mostly) preserves previous CustomOp behavior
# Necessary on ROCm because it's common that users
# enable rms_norm to use the aiter kernel.
# Aiter rms norm perform best when CUDA Graph capture is enabled.
# TODO(luka/TJ) remove env vars completely
if (
cc.is_custom_op_enabled("rms_norm")
cc.cudagraph_mode != CUDAGraphMode.NONE
and envs.VLLM_ROCM_USE_AITER
and envs.VLLM_ROCM_USE_AITER_RMSNORM
):
@@ -962,7 +950,9 @@ class RocmPlatform(Platform):
else:
rms_norm = default
return IrOpPriorityConfig.with_default(default, rms_norm=rms_norm)
return IrOpPriorityConfig.with_default(
default, rms_norm=rms_norm, fused_add_rms_norm=rms_norm
)
@classmethod
@with_amdsmi_context