[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
@@ -29,7 +29,6 @@ from vllm.platforms import current_platform
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from .matcher_utils import (
MatcherFusedAddRMSNorm,
MatcherQuantFP8,
)
@@ -146,9 +145,6 @@ class RMSNormQuantPattern:
assert key in FUSED_OPS, f"unsupported fused rmsnorm+quant op for {key}"
self.FUSED_OP = FUSED_OPS[key]
if key.fused_add:
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
self.quant_matcher = MatcherQuantFP8(
key.quant,
has_col_major_scales=has_col_major_scales,
@@ -231,7 +227,9 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
residual: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, _ = self.quant_matcher(result_rms, scale)
return result, residual
@@ -261,8 +259,9 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
return at[1], at[2]
inputs = [
# input, weight, residual
*self.rmsnorm_matcher.inputs(),
empty_bf16(5, 16), # input
empty_bf16(16), # weight
empty_bf16(5, 16), # residual
self.quant_matcher.inputs()[1], # scale
]
@@ -311,7 +310,9 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
residual: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result = torch.empty(
result_rms.shape,
device=result_rms.device,
@@ -366,12 +367,17 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
# result, residual, scale
return at[1], at[3], at[2]
scale = self.quant_matcher.empty_f32(1, 1)
inputs = [
empty_bf16(5, 16), # input
empty_bf16(16), # weight
empty_bf16(5, 16), # residual
self.quant_matcher.empty_f32(1, 1), # scale
]
pm.register_replacement(
pattern,
replacement,
self.rmsnorm_matcher.inputs() + [scale],
inputs,
pm.fwd_only,
pm_pass,
extra_check=_rms_input_weight_dtype_match,
@@ -552,7 +558,9 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
def pattern(
input: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, scale = self.quant_matcher(result_rms)
return result, residual, scale
@@ -580,10 +588,16 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
# result, residual, scale
return at[1], at[3], at[2]
inputs = [
empty_bf16(5, 16), # input
empty_bf16(16), # weight
empty_bf16(5, 16), # residual
]
pm.register_replacement(
pattern,
replacement,
self.rmsnorm_matcher.inputs(),
inputs,
pm.fwd_only,
pm_pass,
extra_check=_rms_input_weight_dtype_match,