[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)

Signed-off-by: Luka Govedič <lgovedic@redhat.com>
Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
Luka Govedič
2026-05-01 23:41:15 -04:00
committed by GitHub
parent 3e49479c4b
commit d58c42e19c
44 changed files with 2833 additions and 779 deletions
+19
View File
@@ -50,6 +50,7 @@ from .partition_rules import (
should_split,
)
from .passes.inductor_pass import InductorPass, pass_context
from .passes.ir.inplace_functionalization import VllmIRInplaceFunctionalizationPass
from .passes.pass_manager import PostGradPassManager
logger = init_logger(__name__)
@@ -926,6 +927,24 @@ class VllmBackend:
return standalone_compile_artifacts, sym_shape_indices_map, returns_tuple_map
def configure_post_pass(self) -> None:
# TODO proper PassManager?
pre_grad_pass_key = "pre_grad_custom_pass"
assert self.pass_key != pre_grad_pass_key
assert pre_grad_pass_key not in self.inductor_config
self.inductor_config[pre_grad_pass_key] = VllmIRInplaceFunctionalizationPass(
self.vllm_config
)
# Make sure pre_grad_custom_pass is not pickled
# as part of AOTAutograd built-in cache key
# TODO(luka) is there a cleaner way to do this
import torch._inductor.config as inductor_config
ignore = inductor_config._cache_config_ignore_prefix + [pre_grad_pass_key]
assert "_cache_config_ignore_prefix" not in self.inductor_config
self.inductor_config["_cache_config_ignore_prefix"] = ignore
# Configure the (nominally post-grad) pass manager
self.pass_manager.configure(self.vllm_config)
# Post-grad custom passes are run using the post_grad_custom_post_pass
@@ -40,7 +40,7 @@ from ..vllm_inductor_pass import (
VllmPatternMatcherPass,
VllmPatternReplacement,
)
from .matcher_utils import MatcherFusedAddRMSNorm, MatcherQuantFP8
from .matcher_utils import MatcherQuantFP8
FP8_DTYPE = current_platform.fp8_dtype()
@@ -356,10 +356,11 @@ class AllReduceFusedAddRMSNormPattern(BasePattern):
super().__init__(dtype, device)
self.epsilon = epsilon
self.allreduce_params = allreduce_params
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
def get_inputs(self) -> list[torch.Tensor]:
input, residual, weight = self.rmsnorm_matcher.inputs()
input = self.empty(5, 16)
residual = self.empty(5, 16)
weight = self.empty(16)
# input goes through allreduce first, always 16-bit
return [residual, input.to(self.dtype), weight]
@@ -369,7 +370,9 @@ class AllReduceFusedAddRMSNormPattern(BasePattern):
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
rms, residual = vllm.ir.ops.fused_add_rms_norm(
allreduce_output, residual, weight, self.epsilon
)
return rms, residual
def replacement(
@@ -503,11 +506,12 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
self.allreduce_params = allreduce_params
self.quant_dtype = torch.float8_e4m3fn
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
self.quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
def get_inputs(self) -> list[torch.Tensor]:
input, residual, weight = self.rmsnorm_matcher.inputs()
input = self.empty(5, 16)
residual = self.empty(5, 16)
weight = self.empty(16)
_, scale = self.quant_matcher.inputs()
# input goes through allreduce first, always 16-bit
@@ -521,7 +525,9 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms, res = self.rmsnorm_matcher(allreduce_output, weight, residual)
rms, res = vllm.ir.ops.fused_add_rms_norm(
allreduce_output, residual, weight, self.epsilon
)
quant, _ = self.quant_matcher(rms, scale)
return quant, res
@@ -668,7 +674,6 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
super().__init__(dtype, device)
self.epsilon = epsilon
self.allreduce_params = allreduce_params
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
def get_inputs(self) -> list[torch.Tensor]:
input = torch.empty([16, 16], device=self.device, dtype=self.dtype)
@@ -700,7 +705,9 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
input_global_scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
rms, residual = vllm.ir.ops.fused_add_rms_norm(
allreduce_output, residual, weight, self.epsilon
)
quant_out_tuple = auto_functionalized(
STATIC_FP4_QUANT_OP,
input=rms,
@@ -955,15 +962,11 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
super().__init__(dtype, device)
self.epsilon = epsilon
self.dtype = dtype
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(
epsilon, match_rocm_aiter=use_aiter_rmsnorm
)
self.FUSED_AR_RMSNORM_OP = rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()
def get_inputs(self) -> list[torch.Tensor]:
input, residual, weight = self.rmsnorm_matcher.inputs()
return [residual, input.to(self.dtype), weight]
# input, residual, weight
return [self.empty(5, 16), self.empty(5, 16), self.empty(16)]
@property
def pattern(self):
@@ -971,8 +974,9 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]:
allreduce_output = tensor_model_parallel_all_reduce(input)
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
rms, residual = vllm.ir.ops.fused_add_rms_norm(
allreduce_output, residual, weight, self.epsilon
)
return rms, residual
return _pattern
@@ -10,7 +10,6 @@ from torch._ops import OpOverload
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config import get_current_vllm_config
from vllm.model_executor.layers.activation import SiluAndMul
from vllm.model_executor.layers.layernorm import RMSNorm
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
from vllm.model_executor.layers.quantization.utils.quant_utils import (
GroupShape,
@@ -159,72 +158,6 @@ class MatcherRotaryEmbedding(MatcherCustomOp):
return result
class MatcherFusedAddRMSNorm(MatcherCustomOp):
def __init__(
self,
epsilon: float,
enabled: bool | None = None,
match_rocm_aiter: bool = False,
) -> None:
if enabled is None:
enabled = RMSNorm.enabled()
super().__init__(enabled)
self.epsilon = epsilon
self.match_rocm_aiter = match_rocm_aiter
self._rmsnorm_op = RMS_ADD_OP
if match_rocm_aiter:
self._rmsnorm_op = rocm_aiter_ops.get_rmsnorm_fused_add_op()
def inputs(self) -> list[torch.Tensor]:
input = self.empty(5, 16) if self.enabled else self.empty_f32(5, 16)
weight = self.empty(16)
residual = self.empty(5, 16)
return [input, weight, residual]
def forward_rocm_aiter(
self,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
return self._rmsnorm_op( # type: ignore[no-any-return]
x=input, residual=residual, weight=weight, variance_epsilon=self.epsilon
)
def forward_custom(
self,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
if self.match_rocm_aiter:
return self.forward_rocm_aiter(input, weight, residual)
_, result, residual = auto_functionalized(
self._rmsnorm_op,
input=input,
residual=residual,
weight=weight,
epsilon=self.epsilon,
)
return result, residual
def forward_native(
self,
input: torch.Tensor,
weight: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
result: tuple[torch.Tensor, torch.Tensor] = RMSNorm.forward_static(
input, self.epsilon, input.size(-1), self.model_dtype, weight, residual
)
return result
class MatcherQuantFP8(MatcherCustomOp):
def __init__(
self,
@@ -29,7 +29,6 @@ from vllm.platforms import current_platform
from ..inductor_pass import enable_fake_mode
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from .matcher_utils import (
MatcherFusedAddRMSNorm,
MatcherQuantFP8,
)
@@ -146,9 +145,6 @@ class RMSNormQuantPattern:
assert key in FUSED_OPS, f"unsupported fused rmsnorm+quant op for {key}"
self.FUSED_OP = FUSED_OPS[key]
if key.fused_add:
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
self.quant_matcher = MatcherQuantFP8(
key.quant,
has_col_major_scales=has_col_major_scales,
@@ -231,7 +227,9 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
residual: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, _ = self.quant_matcher(result_rms, scale)
return result, residual
@@ -261,8 +259,9 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
return at[1], at[2]
inputs = [
# input, weight, residual
*self.rmsnorm_matcher.inputs(),
empty_bf16(5, 16), # input
empty_bf16(16), # weight
empty_bf16(5, 16), # residual
self.quant_matcher.inputs()[1], # scale
]
@@ -311,7 +310,9 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
residual: torch.Tensor,
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result = torch.empty(
result_rms.shape,
device=result_rms.device,
@@ -366,12 +367,17 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
# result, residual, scale
return at[1], at[3], at[2]
scale = self.quant_matcher.empty_f32(1, 1)
inputs = [
empty_bf16(5, 16), # input
empty_bf16(16), # weight
empty_bf16(5, 16), # residual
self.quant_matcher.empty_f32(1, 1), # scale
]
pm.register_replacement(
pattern,
replacement,
self.rmsnorm_matcher.inputs() + [scale],
inputs,
pm.fwd_only,
pm_pass,
extra_check=_rms_input_weight_dtype_match,
@@ -552,7 +558,9 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
def pattern(
input: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, scale = self.quant_matcher(result_rms)
return result, residual, scale
@@ -580,10 +588,16 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
# result, residual, scale
return at[1], at[3], at[2]
inputs = [
empty_bf16(5, 16), # input
empty_bf16(16), # weight
empty_bf16(5, 16), # residual
]
pm.register_replacement(
pattern,
replacement,
self.rmsnorm_matcher.inputs(),
inputs,
pm.fwd_only,
pm_pass,
extra_check=_rms_input_weight_dtype_match,
@@ -29,7 +29,6 @@ from ..vllm_inductor_pass import (
VllmPatternReplacement,
)
from .matcher_utils import (
MatcherFusedAddRMSNorm,
MatcherQuantFP8,
MatcherSiluAndMul,
)
@@ -49,10 +48,6 @@ class AiterRMSNormQuantPattern:
self.quant_dtype = key.quant.dtype
self.device = torch.device("cuda")
if key.fused_add:
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(
epsilon, match_rocm_aiter=True
)
self.quant_matcher = MatcherQuantFP8(
key.quant,
match_rocm_aiter=match_aiter_quant,
@@ -145,7 +140,9 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
weight: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual_out = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, scale = self.quant_matcher(result_rms)
return result, residual_out, scale
@@ -163,10 +160,16 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
return result[0], result[1], result[2]
inputs = [
self.empty(5, 16), # input
self.empty(16), # weight
self.empty(5, 16), # residual
]
pm.register_replacement(
pattern,
replacement,
self.rmsnorm_matcher.inputs(),
inputs,
pm.fwd_only,
pm_pass,
)
@@ -258,7 +261,9 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
weight: torch.Tensor,
residual: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
result_rms, residual_out = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
result, scale = self.quant_matcher(result_rms)
return result, residual_out, scale
@@ -279,9 +284,13 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
# result, scale, residual
return at[0], at[1], at[2]
pm.register_replacement(
pattern, replacement, self.rmsnorm_matcher.inputs(), pm.fwd_only, pm_pass
)
inputs = [
self.empty(5, 16), # input
self.empty(16), # weight
self.empty(5, 16), # residual
]
pm.register_replacement(pattern, replacement, inputs, pm.fwd_only, pm_pass)
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
@@ -420,12 +429,15 @@ class AddAiterRMSNormPadPattern:
self.epsilon = epsilon
self.hidden_size = hidden_size
self.x_pad_to_multiple = x_pad_to_multiple
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon, match_rocm_aiter=True)
def get_inputs(self) -> list[torch.Tensor]:
input, weight, residual = self.rmsnorm_matcher.inputs()
router_weight = torch.empty([8, 16], dtype=weight.dtype, device=weight.device)
router_bias = torch.empty([8], dtype=weight.dtype, device=weight.device)
device = torch.device("cuda")
dtype = torch.bfloat16
input = torch.empty(5, 16, dtype=dtype, device=device)
weight = torch.empty(16, dtype=dtype, device=device)
residual = torch.empty(5, 16, dtype=dtype, device=device)
router_weight = torch.empty([8, 16], dtype=dtype, device=device)
router_bias = torch.empty([8], dtype=dtype, device=device)
return [input, weight, residual, router_weight, router_bias]
def register(self, pm_pass: PatternMatcherPass) -> None:
@@ -439,7 +451,9 @@ class AddAiterRMSNormPadPattern:
pad_size = self.x_pad_to_multiple - (
self.hidden_size % self.x_pad_to_multiple
)
result_rms, residual_out = self.rmsnorm_matcher(input, weight, residual)
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
input, residual, weight, self.epsilon
)
router_logits = torch.ops.vllm.rocm_unquantized_gemm(
result_rms, router_weight, router_bias
)
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
from ..inductor_pass import enable_fake_mode
from ..utility.noop_elimination import NoOpEliminationPass
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
from .matcher_utils import MatcherFusedAddRMSNorm, MatcherQuantFP8
from .matcher_utils import MatcherQuantFP8
logger = init_logger(__name__)
@@ -174,7 +174,6 @@ class FirstAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
def __init__(self, epsilon: float, dtype: torch.dtype, device: str | None) -> None:
super().__init__(epsilon, dtype, device)
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
def get_inputs(self) -> list[torch.Tensor]:
mm_1 = torch.empty([4, 4], device=self.device, dtype=self.dtype)
@@ -195,7 +194,9 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
rms_norm_weights: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
all_reduce = self._all_reduce(mm_1)
rmsnorm = self.rmsnorm_matcher(all_reduce, rms_norm_weights, residual)
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
all_reduce, residual, rms_norm_weights, self.epsilon
)
return rmsnorm[0], rmsnorm[1]
def replacement(
@@ -208,7 +209,9 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
# once the seqpar pattern with the previous rmsnorm is replaced
reduce_scatter = self._reduce_scatter(mm_1)
residual = residual[0 : reduce_scatter.size(0), ...]
rmsnorm = self.rmsnorm_matcher(reduce_scatter, rms_norm_weights, residual)
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
reduce_scatter, residual, rms_norm_weights, self.epsilon
)
all_gather = self._all_gather(rmsnorm[0])
# shape of residual changes but that's fine,
# next node is already slicing it, now becomes a noop
@@ -271,7 +274,6 @@ class FirstAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
def __init__(self, epsilon: float, dtype: torch.dtype, device: str | None) -> None:
super().__init__(epsilon, dtype, device)
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
self.quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
def get_inputs(self) -> list[torch.Tensor]:
@@ -290,8 +292,8 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
scale: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
all_reduce = self._all_reduce(mm_1)
rms, residual_out = self.rmsnorm_matcher(
all_reduce, rms_norm_weights, residual
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
all_reduce, residual, rms_norm_weights, self.epsilon
)
quant, _ = self.quant_matcher(rms, scale)
return quant, residual_out
@@ -308,8 +310,8 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
# once the seqpar pattern with the previous rmsnorm is replaced
reduce_scatter = self._reduce_scatter(mm_1)
residual = residual[0 : reduce_scatter.size(0), ...]
rms, residual_out = self.rmsnorm_matcher(
reduce_scatter, rms_norm_weights, residual
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
reduce_scatter, residual, rms_norm_weights, self.epsilon
)
quant, _ = self.quant_matcher(rms, scale)
all_gather = self._all_gather(quant)
+3
View File
@@ -30,6 +30,9 @@ class PassContext:
def __init__(self, compile_range: Range):
self.compile_range: Range = compile_range
# set of arg indices
self.donated_input_ids: set[int] = set()
def get_pass_context() -> PassContext:
"""Get the current pass context."""
@@ -0,0 +1,126 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import torch
from torch import fx
from torch._higher_order_ops.auto_functionalize import auto_functionalized
from torch._higher_order_ops.triton_kernel_wrap import TritonKernelWrapperFunctional
from torch._ops import HigherOrderOperator, OpOverload
from vllm.config import VllmConfig
from vllm.logger import init_logger
from ..fx_utils import is_func
from ..inductor_pass import get_pass_context
from ..vllm_inductor_pass import VllmInductorPass
logger = init_logger(__name__)
def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
if user.op == "output":
return False
if is_func(user, auto_functionalized):
# While autofunc writes to the node,
# this is a follow-up use we're not interested in.
# It is also guaranteed to be the final use,
# as auto_functionalized returns the tensor back for follow-up use.
return False
elif user.op == "call_function" and isinstance(user.target, HigherOrderOperator):
# By default, be conservative, assume this could be a write
# (except functional HOPs)
return not isinstance(user.target, TritonKernelWrapperFunctional)
assert isinstance(user.target, OpOverload), (
f"{node=} {user=} {user.op=} {user.target=}"
)
schema = user.target._schema
assert len(user.args) <= len(schema.arguments)
for i, arg in enumerate(user.args):
# Only interested in writes to node
if arg is not node:
continue
# If not a write, next arg could be
if schema.arguments[i].is_write:
return True
# No writes found
return False
class UnsafeCloneEliminationPass(VllmInductorPass):
"""
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
contents of non-donated graph inputs.
It is "unsafe" because it does not (yet) take aliasing into account. Solving
aliasing is an open problem, so this pass intends to support known vLLM cases
and not guarantee soundness on general graphs. In the future, this pass will likely
support basic forms of aliasing to handle simple views (e.g. qkv -> q,k,v).
"""
def __init__(self, vllm_config: VllmConfig) -> None:
super().__init__(vllm_config)
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
count = 0
node_to_idx = {node: i for i, node in enumerate(graph.nodes)}
pass_context = get_pass_context()
donated_input_ids = pass_context.donated_input_ids
logger.debug("Donated input ids: %s", donated_input_ids)
for node in graph.nodes:
if not is_func(node, torch.ops.aten.clone.default):
continue
original_node = node.args[0]
assert isinstance(original_node, fx.Node)
# Clone needs to be preserved if node is getting written to and
# the old value is used again.
# This could only happen if an inplace implementation was lowered.
# Then node (the clone) will have one write.
# TODO(luka) hopefully this can be removed once we lower functional graphs.
write_idxs = [
node_to_idx[u] for u in node.users if user_writes_to_node(u, node)
]
assert len(write_idxs) in (0, 1)
if write_idxs:
# Check if a user of original_node occurs after a write
write_idx = write_idxs[0]
if any(
node_to_idx[orig_user] > write_idx
for orig_user in original_node.users
):
logger.debug(
"Clone removal not possible, "
"original_node=%s used after mutation on node=%s",
original_node,
node,
)
continue
# Check if a node is a (non-donated) graph input
if (
original_node.op == "placeholder"
and node_to_idx[original_node] not in donated_input_ids
):
logger.debug(
"Graph input %s not donated, cannot eliminate its clone",
original_node,
)
continue
logger.debug(
"Node %s is a redundant clone node of %s, removing it",
node,
original_node,
)
node.replace_all_uses_with(original_node)
graph.erase_node(node)
count += 1
logger.debug("CloneCleanupPass removed %d clone nodes", count)
@@ -0,0 +1,101 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections import defaultdict
from torch import fx
from torch._inductor.pattern_matcher import (
PatternMatcherPass,
)
from vllm.config import VllmConfig
from vllm.logger import init_logger
from ..inductor_pass import get_pass_context
from ..vllm_inductor_pass import VllmInductorPass
from .lowering_pass import get_ir_op
from .utils import overload_or_default
logger = init_logger(__name__)
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
"""
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
The maybe_inplace overloads have the same signature as the default overload
so the pass simply replaces the called overload.
That makes the graph properly functional.
The pass also validates that activations passed to maybe_inplace have no later
uses in the graph: they are donated to the maybe_inplace op call,
and their contents are not defined afterward.
This pass operates pre-AOTAutograd,
so it must handle non-normalized and non-functional IR.
"""
def __init__(self, vllm_config: VllmConfig) -> None:
super().__init__(vllm_config)
self.patterns = PatternMatcherPass(self.pass_name)
self.functionalized_ops: dict[str, int] = defaultdict(lambda: 0)
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
# clear at the beginning instead of end, so that tests can inspect
self.functionalized_ops.clear()
assert graph.owning_module is not None
node_to_idx = {node: i for i, node in enumerate(graph.nodes)}
# Pass donated input via vLLM's pass context
pass_context = get_pass_context()
pass_context.donated_input_ids = set[int]()
for node in graph.nodes:
if (ir_op := get_ir_op(node)) is None:
continue
op_overload = overload_or_default(node.target)
overload_name = op_overload._overloadname
if overload_name != "maybe_inplace":
assert overload_name == "default", (
f"Found overload {overload_name} for op {ir_op.name}, "
f"expected maybe_inplace or default"
)
continue
# must have maybe_inplace overload and allow_inplace
assert ir_op.allow_inplace and hasattr(ir_op, "maybe_inplace")
# Check that activation inputs are not used after this op
for arg_idx in ir_op.activation_indices:
arg = node.args[arg_idx]
assert isinstance(arg, fx.Node), "Activation inputs must be fx.Node"
for user in arg.users:
if node_to_idx[user] > node_to_idx[node]:
raise ValueError(
f"Input {arg} to maybe_inplace node {node} "
f"is used again after the node. "
f"This is not allowed; activation inputs to maybe_inplace "
f"ops are donated to the op, meaning their memory may be "
f"recycled for outputs.\n\n"
f"To preserve the inputs, use the default overload or "
f"clone them manually beforehand."
)
if arg.op == "placeholder":
# Graph input that maybe_inplace might modify.
# Mark it so downstream passes know it's donated.
# TODO(luka) store in placeholder node meta once supported
pass_context.donated_input_ids.add(node_to_idx[arg])
# Same signature, just replace the overload that's called.
node.target = ir_op.torch_op
self.functionalized_ops[ir_op.name] += 1
count = sum(self.functionalized_ops.values())
ops = ",".join(self.functionalized_ops.keys())
logger.debug("Donated input IDs: %s", pass_context.donated_input_ids)
logger.debug(
"%s functionalized %d vLLM IR nodes for op(s) %s",
self.pass_name,
count,
ops,
)
+9 -35
View File
@@ -10,7 +10,6 @@ from torch._inductor.pattern_matcher import (
PatternMatcherPass,
register_graph_pattern,
)
from torch._ops import OpOverload, OpOverloadPacket
from vllm.config import VllmConfig
from vllm.ir.op import IrOp
@@ -18,41 +17,11 @@ from vllm.logger import init_logger
from vllm.logging_utils import lazy
from ..vllm_inductor_pass import VllmInductorPass
from .utils import get_ir_op
logger = init_logger(__name__)
def get_default_overload(op: OpOverload | OpOverloadPacket) -> OpOverload:
if isinstance(op, OpOverloadPacket):
return op.default
assert isinstance(op, OpOverload), "Expected an OpOverload or OpOverloadPacket"
return op
def get_ir_op(node: fx.Node) -> IrOp | None:
if node.op != "call_function":
return None
if not isinstance(node.target, (OpOverload, OpOverloadPacket)):
return None
op_overload = get_default_overload(node.target)
if op_overload.namespace != "vllm_ir":
return None
op_name = op_overload._opname
if op_name not in IrOp.registry:
logger.warning(
"Unknown vLLM IR op %s, there's likely an issue with torch registration, "
"or a torch custom op was registered in the vllm_ir namespace by mistake.",
op_name,
)
return None
ir_op = IrOp.registry[op_name]
return ir_op
class VllmIRLoweringPass(VllmInductorPass):
"""
This pass lowers vLLM IR ops to their implementations the priority list.
@@ -76,7 +45,7 @@ class VllmIRLoweringPass(VllmInductorPass):
assert len(match.nodes) == 1, "Expected single node match"
node = match.nodes[0]
ir_op = get_ir_op(node)
ir_op = get_ir_op(node) # TODO is node.target always an overload?
assert ir_op is not None, "Expected vLLM IR op"
assert not node.kwargs # I think there should never be kwargs here
@@ -86,13 +55,18 @@ class VllmIRLoweringPass(VllmInductorPass):
self.selected_impls[ir_op.name][node.name] = ir_op_impl.provider
# replace_by_example wants node args, not the fake tensors
# use func_impl_fn to properly handle in-place implementations
# TODO(luka): Use aot_export_module to get functionalized graph
# TODO(luka): Cache the fx_replacement to avoid re-tracing the same impl
# Defaults not present on node.args but required for replacement tracing
bound_args = ir_op._py_signature.bind(*node.args)
bound_args.apply_defaults()
match.replace_by_example(ir_op_impl.impl_fn, bound_args.args)
# It is not safe to run functional passes (like DCE) on the replacements
# as they might not be functional.
match.replace_by_example(
ir_op_impl.func_impl_fn, bound_args.args, run_functional_passes=False
)
@VllmInductorPass.time_and_log
def __call__(self, graph: fx.Graph) -> None:
@@ -136,7 +110,7 @@ class VllmIRLoweringPass(VllmInductorPass):
if failed_nodes or failed_ops:
logger.warning("Failed to lower vLLM IR ops: %s", ",".join(failed_ops))
logger.warning("Full node list: %s", failed_nodes)
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
def uuid(self) -> str:
"""
+40
View File
@@ -0,0 +1,40 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from torch import fx
from torch._ops import OpOverload, OpOverloadPacket
from vllm.ir.op import IrOp
from vllm.logger import init_logger
logger = init_logger(__name__)
def overload_or_default(op: OpOverload | OpOverloadPacket) -> OpOverload:
if isinstance(op, OpOverloadPacket):
return op.default
assert isinstance(op, OpOverload), "Expected an OpOverload or OpOverloadPacket"
return op
def get_ir_op(node: fx.Node) -> IrOp | None:
if node.op != "call_function":
return None
if not isinstance(node.target, (OpOverload, OpOverloadPacket)):
return None
op_overload = overload_or_default(node.target)
if op_overload.namespace != "vllm_ir":
return None
op_name = op_overload._opname
if op_name not in IrOp.registry:
logger.warning(
"Unknown vLLM IR op %s, there's likely an issue with torch registration, "
"or a torch custom op was registered in the vllm_ir namespace by mistake.",
op_name,
)
return None
ir_op = IrOp.registry[op_name]
return ir_op
+7 -1
View File
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
from vllm.platforms import current_platform
from vllm.utils.system_utils import set_env_var
from .ir.clone_elimination import UnsafeCloneEliminationPass
from .ir.lowering_pass import VllmIRLoweringPass
from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
@@ -115,6 +116,8 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
# DCE handles mutating ops correctly as well.
self.ir_lowering(graph)
VllmInductorPass.dump_prefix += 1
self.clone_elimination(graph)
VllmInductorPass.dump_prefix += 1
# clean up after lowering again
self.post_cleanup(graph)
@@ -149,11 +152,12 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes += [MiniMaxQKNormPass(config)]
if self.pass_config.fuse_norm_quant:
self.passes += [RMSNormQuantFusionPass(config)]
if rocm_aiter_ops.is_enabled():
self.passes += [
RocmAiterRMSNormQuantFusionPass(config),
]
self.passes += [RMSNormQuantFusionPass(config)]
if self.pass_config.fuse_act_quant:
self.passes += [ActivationQuantFusionPass(config)]
if rocm_aiter_ops.is_enabled():
@@ -179,6 +183,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
self.passes += [QKNormRoPEFusionPass(config)]
self.ir_lowering = VllmIRLoweringPass(config)
self.clone_elimination = UnsafeCloneEliminationPass(config)
self.post_cleanup = PostCleanupPass(config)
self.fix_functionalization = FixFunctionalizationPass(config)
@@ -200,6 +205,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
passes.append(self.post_cleanup.uuid())
passes.append(self.ir_lowering.uuid())
passes.append(self.clone_elimination.uuid())
passes.append(self.post_cleanup.uuid())
passes.append(self.fix_functionalization.uuid())