[vLLM IR] 2/N fused_add_rms_norm and maybe_inplace overload (#36823)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Signed-off-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
@@ -50,6 +50,7 @@ from .partition_rules import (
|
||||
should_split,
|
||||
)
|
||||
from .passes.inductor_pass import InductorPass, pass_context
|
||||
from .passes.ir.inplace_functionalization import VllmIRInplaceFunctionalizationPass
|
||||
from .passes.pass_manager import PostGradPassManager
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -926,6 +927,24 @@ class VllmBackend:
|
||||
return standalone_compile_artifacts, sym_shape_indices_map, returns_tuple_map
|
||||
|
||||
def configure_post_pass(self) -> None:
|
||||
# TODO proper PassManager?
|
||||
pre_grad_pass_key = "pre_grad_custom_pass"
|
||||
assert self.pass_key != pre_grad_pass_key
|
||||
assert pre_grad_pass_key not in self.inductor_config
|
||||
self.inductor_config[pre_grad_pass_key] = VllmIRInplaceFunctionalizationPass(
|
||||
self.vllm_config
|
||||
)
|
||||
|
||||
# Make sure pre_grad_custom_pass is not pickled
|
||||
# as part of AOTAutograd built-in cache key
|
||||
# TODO(luka) is there a cleaner way to do this
|
||||
import torch._inductor.config as inductor_config
|
||||
|
||||
ignore = inductor_config._cache_config_ignore_prefix + [pre_grad_pass_key]
|
||||
assert "_cache_config_ignore_prefix" not in self.inductor_config
|
||||
self.inductor_config["_cache_config_ignore_prefix"] = ignore
|
||||
|
||||
# Configure the (nominally post-grad) pass manager
|
||||
self.pass_manager.configure(self.vllm_config)
|
||||
|
||||
# Post-grad custom passes are run using the post_grad_custom_post_pass
|
||||
|
||||
@@ -40,7 +40,7 @@ from ..vllm_inductor_pass import (
|
||||
VllmPatternMatcherPass,
|
||||
VllmPatternReplacement,
|
||||
)
|
||||
from .matcher_utils import MatcherFusedAddRMSNorm, MatcherQuantFP8
|
||||
from .matcher_utils import MatcherQuantFP8
|
||||
|
||||
FP8_DTYPE = current_platform.fp8_dtype()
|
||||
|
||||
@@ -356,10 +356,11 @@ class AllReduceFusedAddRMSNormPattern(BasePattern):
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.allreduce_params = allreduce_params
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input, residual, weight = self.rmsnorm_matcher.inputs()
|
||||
input = self.empty(5, 16)
|
||||
residual = self.empty(5, 16)
|
||||
weight = self.empty(16)
|
||||
|
||||
# input goes through allreduce first, always 16-bit
|
||||
return [residual, input.to(self.dtype), weight]
|
||||
@@ -369,7 +370,9 @@ class AllReduceFusedAddRMSNormPattern(BasePattern):
|
||||
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
|
||||
rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
allreduce_output, residual, weight, self.epsilon
|
||||
)
|
||||
return rms, residual
|
||||
|
||||
def replacement(
|
||||
@@ -503,11 +506,12 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
self.allreduce_params = allreduce_params
|
||||
self.quant_dtype = torch.float8_e4m3fn
|
||||
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
|
||||
self.quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input, residual, weight = self.rmsnorm_matcher.inputs()
|
||||
input = self.empty(5, 16)
|
||||
residual = self.empty(5, 16)
|
||||
weight = self.empty(16)
|
||||
_, scale = self.quant_matcher.inputs()
|
||||
|
||||
# input goes through allreduce first, always 16-bit
|
||||
@@ -521,7 +525,9 @@ class AllReduceFusedAddRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms, res = self.rmsnorm_matcher(allreduce_output, weight, residual)
|
||||
rms, res = vllm.ir.ops.fused_add_rms_norm(
|
||||
allreduce_output, residual, weight, self.epsilon
|
||||
)
|
||||
quant, _ = self.quant_matcher(rms, scale)
|
||||
|
||||
return quant, res
|
||||
@@ -668,7 +674,6 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.allreduce_params = allreduce_params
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input = torch.empty([16, 16], device=self.device, dtype=self.dtype)
|
||||
@@ -700,7 +705,9 @@ class AllReduceFusedAddRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
input_global_scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
|
||||
rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
allreduce_output, residual, weight, self.epsilon
|
||||
)
|
||||
quant_out_tuple = auto_functionalized(
|
||||
STATIC_FP4_QUANT_OP,
|
||||
input=rms,
|
||||
@@ -955,15 +962,11 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
super().__init__(dtype, device)
|
||||
self.epsilon = epsilon
|
||||
self.dtype = dtype
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(
|
||||
epsilon, match_rocm_aiter=use_aiter_rmsnorm
|
||||
)
|
||||
self.FUSED_AR_RMSNORM_OP = rocm_aiter_ops.get_fused_allreduce_rmsnorm_op()
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input, residual, weight = self.rmsnorm_matcher.inputs()
|
||||
|
||||
return [residual, input.to(self.dtype), weight]
|
||||
# input, residual, weight
|
||||
return [self.empty(5, 16), self.empty(5, 16), self.empty(16)]
|
||||
|
||||
@property
|
||||
def pattern(self):
|
||||
@@ -971,8 +974,9 @@ class AiterAllreduceFusedAddRMSNormPattern(BasePattern, VllmPatternReplacement):
|
||||
residual: torch.Tensor, input: torch.Tensor, weight: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
allreduce_output = tensor_model_parallel_all_reduce(input)
|
||||
rms, residual = self.rmsnorm_matcher(allreduce_output, weight, residual)
|
||||
|
||||
rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
allreduce_output, residual, weight, self.epsilon
|
||||
)
|
||||
return rms, residual
|
||||
|
||||
return _pattern
|
||||
|
||||
@@ -10,7 +10,6 @@ from torch._ops import OpOverload
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config import get_current_vllm_config
|
||||
from vllm.model_executor.layers.activation import SiluAndMul
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
GroupShape,
|
||||
@@ -159,72 +158,6 @@ class MatcherRotaryEmbedding(MatcherCustomOp):
|
||||
return result
|
||||
|
||||
|
||||
class MatcherFusedAddRMSNorm(MatcherCustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
epsilon: float,
|
||||
enabled: bool | None = None,
|
||||
match_rocm_aiter: bool = False,
|
||||
) -> None:
|
||||
if enabled is None:
|
||||
enabled = RMSNorm.enabled()
|
||||
|
||||
super().__init__(enabled)
|
||||
self.epsilon = epsilon
|
||||
self.match_rocm_aiter = match_rocm_aiter
|
||||
|
||||
self._rmsnorm_op = RMS_ADD_OP
|
||||
|
||||
if match_rocm_aiter:
|
||||
self._rmsnorm_op = rocm_aiter_ops.get_rmsnorm_fused_add_op()
|
||||
|
||||
def inputs(self) -> list[torch.Tensor]:
|
||||
input = self.empty(5, 16) if self.enabled else self.empty_f32(5, 16)
|
||||
weight = self.empty(16)
|
||||
residual = self.empty(5, 16)
|
||||
return [input, weight, residual]
|
||||
|
||||
def forward_rocm_aiter(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
return self._rmsnorm_op( # type: ignore[no-any-return]
|
||||
x=input, residual=residual, weight=weight, variance_epsilon=self.epsilon
|
||||
)
|
||||
|
||||
def forward_custom(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.match_rocm_aiter:
|
||||
return self.forward_rocm_aiter(input, weight, residual)
|
||||
|
||||
_, result, residual = auto_functionalized(
|
||||
self._rmsnorm_op,
|
||||
input=input,
|
||||
residual=residual,
|
||||
weight=weight,
|
||||
epsilon=self.epsilon,
|
||||
)
|
||||
|
||||
return result, residual
|
||||
|
||||
def forward_native(
|
||||
self,
|
||||
input: torch.Tensor,
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
result: tuple[torch.Tensor, torch.Tensor] = RMSNorm.forward_static(
|
||||
input, self.epsilon, input.size(-1), self.model_dtype, weight, residual
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
class MatcherQuantFP8(MatcherCustomOp):
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
@@ -29,7 +29,6 @@ from vllm.platforms import current_platform
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
from .matcher_utils import (
|
||||
MatcherFusedAddRMSNorm,
|
||||
MatcherQuantFP8,
|
||||
)
|
||||
|
||||
@@ -146,9 +145,6 @@ class RMSNormQuantPattern:
|
||||
assert key in FUSED_OPS, f"unsupported fused rmsnorm+quant op for {key}"
|
||||
self.FUSED_OP = FUSED_OPS[key]
|
||||
|
||||
if key.fused_add:
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
|
||||
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
key.quant,
|
||||
has_col_major_scales=has_col_major_scales,
|
||||
@@ -231,7 +227,9 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
|
||||
residual: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
|
||||
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
result, _ = self.quant_matcher(result_rms, scale)
|
||||
|
||||
return result, residual
|
||||
@@ -261,8 +259,9 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
|
||||
return at[1], at[2]
|
||||
|
||||
inputs = [
|
||||
# input, weight, residual
|
||||
*self.rmsnorm_matcher.inputs(),
|
||||
empty_bf16(5, 16), # input
|
||||
empty_bf16(16), # weight
|
||||
empty_bf16(5, 16), # residual
|
||||
self.quant_matcher.inputs()[1], # scale
|
||||
]
|
||||
|
||||
@@ -311,7 +310,9 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
|
||||
residual: torch.Tensor,
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
|
||||
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
result = torch.empty(
|
||||
result_rms.shape,
|
||||
device=result_rms.device,
|
||||
@@ -366,12 +367,17 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
|
||||
# result, residual, scale
|
||||
return at[1], at[3], at[2]
|
||||
|
||||
scale = self.quant_matcher.empty_f32(1, 1)
|
||||
inputs = [
|
||||
empty_bf16(5, 16), # input
|
||||
empty_bf16(16), # weight
|
||||
empty_bf16(5, 16), # residual
|
||||
self.quant_matcher.empty_f32(1, 1), # scale
|
||||
]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
self.rmsnorm_matcher.inputs() + [scale],
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
@@ -552,7 +558,9 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
|
||||
def pattern(
|
||||
input: torch.Tensor, weight: torch.Tensor, residual: torch.Tensor
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms, residual = self.rmsnorm_matcher(input, weight, residual)
|
||||
result_rms, residual = vllm.ir.ops.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
result, scale = self.quant_matcher(result_rms)
|
||||
|
||||
return result, residual, scale
|
||||
@@ -580,10 +588,16 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
|
||||
# result, residual, scale
|
||||
return at[1], at[3], at[2]
|
||||
|
||||
inputs = [
|
||||
empty_bf16(5, 16), # input
|
||||
empty_bf16(16), # weight
|
||||
empty_bf16(5, 16), # residual
|
||||
]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
self.rmsnorm_matcher.inputs(),
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
|
||||
@@ -29,7 +29,6 @@ from ..vllm_inductor_pass import (
|
||||
VllmPatternReplacement,
|
||||
)
|
||||
from .matcher_utils import (
|
||||
MatcherFusedAddRMSNorm,
|
||||
MatcherQuantFP8,
|
||||
MatcherSiluAndMul,
|
||||
)
|
||||
@@ -49,10 +48,6 @@ class AiterRMSNormQuantPattern:
|
||||
self.quant_dtype = key.quant.dtype
|
||||
self.device = torch.device("cuda")
|
||||
|
||||
if key.fused_add:
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(
|
||||
epsilon, match_rocm_aiter=True
|
||||
)
|
||||
self.quant_matcher = MatcherQuantFP8(
|
||||
key.quant,
|
||||
match_rocm_aiter=match_aiter_quant,
|
||||
@@ -145,7 +140,9 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms, residual_out = self.rmsnorm_matcher(input, weight, residual)
|
||||
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
result, scale = self.quant_matcher(result_rms)
|
||||
|
||||
return result, residual_out, scale
|
||||
@@ -163,10 +160,16 @@ class AiterFusedAddRMSNormDynamicQuantPattern(AiterRMSNormQuantPattern):
|
||||
|
||||
return result[0], result[1], result[2]
|
||||
|
||||
inputs = [
|
||||
self.empty(5, 16), # input
|
||||
self.empty(16), # weight
|
||||
self.empty(5, 16), # residual
|
||||
]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
self.rmsnorm_matcher.inputs(),
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
)
|
||||
@@ -258,7 +261,9 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
weight: torch.Tensor,
|
||||
residual: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
result_rms, residual_out = self.rmsnorm_matcher(input, weight, residual)
|
||||
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
result, scale = self.quant_matcher(result_rms)
|
||||
|
||||
return result, residual_out, scale
|
||||
@@ -279,9 +284,13 @@ class AiterFusedAddRMSFp8GroupQuantPattern(AiterRMSNormQuantPattern):
|
||||
# result, scale, residual
|
||||
return at[0], at[1], at[2]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.rmsnorm_matcher.inputs(), pm.fwd_only, pm_pass
|
||||
)
|
||||
inputs = [
|
||||
self.empty(5, 16), # input
|
||||
self.empty(16), # weight
|
||||
self.empty(5, 16), # residual
|
||||
]
|
||||
|
||||
pm.register_replacement(pattern, replacement, inputs, pm.fwd_only, pm_pass)
|
||||
|
||||
|
||||
class RocmAiterRMSNormQuantFusionPass(VllmPatternMatcherPass):
|
||||
@@ -420,12 +429,15 @@ class AddAiterRMSNormPadPattern:
|
||||
self.epsilon = epsilon
|
||||
self.hidden_size = hidden_size
|
||||
self.x_pad_to_multiple = x_pad_to_multiple
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon, match_rocm_aiter=True)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
input, weight, residual = self.rmsnorm_matcher.inputs()
|
||||
router_weight = torch.empty([8, 16], dtype=weight.dtype, device=weight.device)
|
||||
router_bias = torch.empty([8], dtype=weight.dtype, device=weight.device)
|
||||
device = torch.device("cuda")
|
||||
dtype = torch.bfloat16
|
||||
input = torch.empty(5, 16, dtype=dtype, device=device)
|
||||
weight = torch.empty(16, dtype=dtype, device=device)
|
||||
residual = torch.empty(5, 16, dtype=dtype, device=device)
|
||||
router_weight = torch.empty([8, 16], dtype=dtype, device=device)
|
||||
router_bias = torch.empty([8], dtype=dtype, device=device)
|
||||
return [input, weight, residual, router_weight, router_bias]
|
||||
|
||||
def register(self, pm_pass: PatternMatcherPass) -> None:
|
||||
@@ -439,7 +451,9 @@ class AddAiterRMSNormPadPattern:
|
||||
pad_size = self.x_pad_to_multiple - (
|
||||
self.hidden_size % self.x_pad_to_multiple
|
||||
)
|
||||
result_rms, residual_out = self.rmsnorm_matcher(input, weight, residual)
|
||||
result_rms, residual_out = torch.ops.vllm_ir.fused_add_rms_norm(
|
||||
input, residual, weight, self.epsilon
|
||||
)
|
||||
router_logits = torch.ops.vllm.rocm_unquantized_gemm(
|
||||
result_rms, router_weight, router_bias
|
||||
)
|
||||
|
||||
@@ -23,7 +23,7 @@ from vllm.model_executor.layers.quantization.utils.quant_utils import (
|
||||
from ..inductor_pass import enable_fake_mode
|
||||
from ..utility.noop_elimination import NoOpEliminationPass
|
||||
from ..vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
from .matcher_utils import MatcherFusedAddRMSNorm, MatcherQuantFP8
|
||||
from .matcher_utils import MatcherQuantFP8
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -174,7 +174,6 @@ class FirstAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
|
||||
class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
|
||||
def __init__(self, epsilon: float, dtype: torch.dtype, device: str | None) -> None:
|
||||
super().__init__(epsilon, dtype, device)
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
mm_1 = torch.empty([4, 4], device=self.device, dtype=self.dtype)
|
||||
@@ -195,7 +194,9 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
|
||||
rms_norm_weights: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
all_reduce = self._all_reduce(mm_1)
|
||||
rmsnorm = self.rmsnorm_matcher(all_reduce, rms_norm_weights, residual)
|
||||
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
|
||||
all_reduce, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
return rmsnorm[0], rmsnorm[1]
|
||||
|
||||
def replacement(
|
||||
@@ -208,7 +209,9 @@ class MiddleAllReduceRMSNormPattern(_SequenceParallelPatternHelper):
|
||||
# once the seqpar pattern with the previous rmsnorm is replaced
|
||||
reduce_scatter = self._reduce_scatter(mm_1)
|
||||
residual = residual[0 : reduce_scatter.size(0), ...]
|
||||
rmsnorm = self.rmsnorm_matcher(reduce_scatter, rms_norm_weights, residual)
|
||||
rmsnorm = vllm.ir.ops.fused_add_rms_norm(
|
||||
reduce_scatter, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
all_gather = self._all_gather(rmsnorm[0])
|
||||
# shape of residual changes but that's fine,
|
||||
# next node is already slicing it, now becomes a noop
|
||||
@@ -271,7 +274,6 @@ class FirstAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
|
||||
class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
|
||||
def __init__(self, epsilon: float, dtype: torch.dtype, device: str | None) -> None:
|
||||
super().__init__(epsilon, dtype, device)
|
||||
self.rmsnorm_matcher = MatcherFusedAddRMSNorm(epsilon)
|
||||
self.quant_matcher = MatcherQuantFP8(kFp8StaticTensorSym)
|
||||
|
||||
def get_inputs(self) -> list[torch.Tensor]:
|
||||
@@ -290,8 +292,8 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
|
||||
scale: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
all_reduce = self._all_reduce(mm_1)
|
||||
rms, residual_out = self.rmsnorm_matcher(
|
||||
all_reduce, rms_norm_weights, residual
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
all_reduce, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
quant, _ = self.quant_matcher(rms, scale)
|
||||
return quant, residual_out
|
||||
@@ -308,8 +310,8 @@ class MiddleAllReduceRMSNormStaticFP8Pattern(_SequenceParallelPatternHelper):
|
||||
# once the seqpar pattern with the previous rmsnorm is replaced
|
||||
reduce_scatter = self._reduce_scatter(mm_1)
|
||||
residual = residual[0 : reduce_scatter.size(0), ...]
|
||||
rms, residual_out = self.rmsnorm_matcher(
|
||||
reduce_scatter, rms_norm_weights, residual
|
||||
rms, residual_out = vllm.ir.ops.fused_add_rms_norm(
|
||||
reduce_scatter, residual, rms_norm_weights, self.epsilon
|
||||
)
|
||||
quant, _ = self.quant_matcher(rms, scale)
|
||||
all_gather = self._all_gather(quant)
|
||||
|
||||
@@ -30,6 +30,9 @@ class PassContext:
|
||||
def __init__(self, compile_range: Range):
|
||||
self.compile_range: Range = compile_range
|
||||
|
||||
# set of arg indices
|
||||
self.donated_input_ids: set[int] = set()
|
||||
|
||||
|
||||
def get_pass_context() -> PassContext:
|
||||
"""Get the current pass context."""
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import torch
|
||||
from torch import fx
|
||||
from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
from torch._higher_order_ops.triton_kernel_wrap import TritonKernelWrapperFunctional
|
||||
from torch._ops import HigherOrderOperator, OpOverload
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..fx_utils import is_func
|
||||
from ..inductor_pass import get_pass_context
|
||||
from ..vllm_inductor_pass import VllmInductorPass
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def user_writes_to_node(user: fx.Node, node: fx.Node) -> bool:
|
||||
if user.op == "output":
|
||||
return False
|
||||
|
||||
if is_func(user, auto_functionalized):
|
||||
# While autofunc writes to the node,
|
||||
# this is a follow-up use we're not interested in.
|
||||
# It is also guaranteed to be the final use,
|
||||
# as auto_functionalized returns the tensor back for follow-up use.
|
||||
return False
|
||||
elif user.op == "call_function" and isinstance(user.target, HigherOrderOperator):
|
||||
# By default, be conservative, assume this could be a write
|
||||
# (except functional HOPs)
|
||||
return not isinstance(user.target, TritonKernelWrapperFunctional)
|
||||
|
||||
assert isinstance(user.target, OpOverload), (
|
||||
f"{node=} {user=} {user.op=} {user.target=}"
|
||||
)
|
||||
schema = user.target._schema
|
||||
assert len(user.args) <= len(schema.arguments)
|
||||
for i, arg in enumerate(user.args):
|
||||
# Only interested in writes to node
|
||||
if arg is not node:
|
||||
continue
|
||||
|
||||
# If not a write, next arg could be
|
||||
if schema.arguments[i].is_write:
|
||||
return True
|
||||
|
||||
# No writes found
|
||||
return False
|
||||
|
||||
|
||||
class UnsafeCloneEliminationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass removes clone nodes that are no longer needed after vLLM IR lowering.
|
||||
It uses donated_input_ids to eliminate clones of donated graph inputs, preserving
|
||||
contents of non-donated graph inputs.
|
||||
|
||||
It is "unsafe" because it does not (yet) take aliasing into account. Solving
|
||||
aliasing is an open problem, so this pass intends to support known vLLM cases
|
||||
and not guarantee soundness on general graphs. In the future, this pass will likely
|
||||
support basic forms of aliasing to handle simple views (e.g. qkv -> q,k,v).
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
super().__init__(vllm_config)
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
count = 0
|
||||
node_to_idx = {node: i for i, node in enumerate(graph.nodes)}
|
||||
pass_context = get_pass_context()
|
||||
donated_input_ids = pass_context.donated_input_ids
|
||||
logger.debug("Donated input ids: %s", donated_input_ids)
|
||||
|
||||
for node in graph.nodes:
|
||||
if not is_func(node, torch.ops.aten.clone.default):
|
||||
continue
|
||||
|
||||
original_node = node.args[0]
|
||||
assert isinstance(original_node, fx.Node)
|
||||
|
||||
# Clone needs to be preserved if node is getting written to and
|
||||
# the old value is used again.
|
||||
# This could only happen if an inplace implementation was lowered.
|
||||
# Then node (the clone) will have one write.
|
||||
# TODO(luka) hopefully this can be removed once we lower functional graphs.
|
||||
write_idxs = [
|
||||
node_to_idx[u] for u in node.users if user_writes_to_node(u, node)
|
||||
]
|
||||
assert len(write_idxs) in (0, 1)
|
||||
if write_idxs:
|
||||
# Check if a user of original_node occurs after a write
|
||||
write_idx = write_idxs[0]
|
||||
if any(
|
||||
node_to_idx[orig_user] > write_idx
|
||||
for orig_user in original_node.users
|
||||
):
|
||||
logger.debug(
|
||||
"Clone removal not possible, "
|
||||
"original_node=%s used after mutation on node=%s",
|
||||
original_node,
|
||||
node,
|
||||
)
|
||||
continue
|
||||
|
||||
# Check if a node is a (non-donated) graph input
|
||||
if (
|
||||
original_node.op == "placeholder"
|
||||
and node_to_idx[original_node] not in donated_input_ids
|
||||
):
|
||||
logger.debug(
|
||||
"Graph input %s not donated, cannot eliminate its clone",
|
||||
original_node,
|
||||
)
|
||||
continue
|
||||
|
||||
logger.debug(
|
||||
"Node %s is a redundant clone node of %s, removing it",
|
||||
node,
|
||||
original_node,
|
||||
)
|
||||
node.replace_all_uses_with(original_node)
|
||||
graph.erase_node(node)
|
||||
count += 1
|
||||
|
||||
logger.debug("CloneCleanupPass removed %d clone nodes", count)
|
||||
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from collections import defaultdict
|
||||
|
||||
from torch import fx
|
||||
from torch._inductor.pattern_matcher import (
|
||||
PatternMatcherPass,
|
||||
)
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from ..inductor_pass import get_pass_context
|
||||
from ..vllm_inductor_pass import VllmInductorPass
|
||||
from .lowering_pass import get_ir_op
|
||||
from .utils import overload_or_default
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class VllmIRInplaceFunctionalizationPass(VllmInductorPass):
|
||||
"""
|
||||
This pass functionalizes maybe_inplace vLLM IR ops to the default overload.
|
||||
The maybe_inplace overloads have the same signature as the default overload
|
||||
so the pass simply replaces the called overload.
|
||||
That makes the graph properly functional.
|
||||
The pass also validates that activations passed to maybe_inplace have no later
|
||||
uses in the graph: they are donated to the maybe_inplace op call,
|
||||
and their contents are not defined afterward.
|
||||
|
||||
This pass operates pre-AOTAutograd,
|
||||
so it must handle non-normalized and non-functional IR.
|
||||
"""
|
||||
|
||||
def __init__(self, vllm_config: VllmConfig) -> None:
|
||||
super().__init__(vllm_config)
|
||||
self.patterns = PatternMatcherPass(self.pass_name)
|
||||
self.functionalized_ops: dict[str, int] = defaultdict(lambda: 0)
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
# clear at the beginning instead of end, so that tests can inspect
|
||||
self.functionalized_ops.clear()
|
||||
assert graph.owning_module is not None
|
||||
node_to_idx = {node: i for i, node in enumerate(graph.nodes)}
|
||||
|
||||
# Pass donated input via vLLM's pass context
|
||||
pass_context = get_pass_context()
|
||||
pass_context.donated_input_ids = set[int]()
|
||||
|
||||
for node in graph.nodes:
|
||||
if (ir_op := get_ir_op(node)) is None:
|
||||
continue
|
||||
|
||||
op_overload = overload_or_default(node.target)
|
||||
overload_name = op_overload._overloadname
|
||||
if overload_name != "maybe_inplace":
|
||||
assert overload_name == "default", (
|
||||
f"Found overload {overload_name} for op {ir_op.name}, "
|
||||
f"expected maybe_inplace or default"
|
||||
)
|
||||
continue
|
||||
|
||||
# must have maybe_inplace overload and allow_inplace
|
||||
assert ir_op.allow_inplace and hasattr(ir_op, "maybe_inplace")
|
||||
|
||||
# Check that activation inputs are not used after this op
|
||||
for arg_idx in ir_op.activation_indices:
|
||||
arg = node.args[arg_idx]
|
||||
assert isinstance(arg, fx.Node), "Activation inputs must be fx.Node"
|
||||
for user in arg.users:
|
||||
if node_to_idx[user] > node_to_idx[node]:
|
||||
raise ValueError(
|
||||
f"Input {arg} to maybe_inplace node {node} "
|
||||
f"is used again after the node. "
|
||||
f"This is not allowed; activation inputs to maybe_inplace "
|
||||
f"ops are donated to the op, meaning their memory may be "
|
||||
f"recycled for outputs.\n\n"
|
||||
f"To preserve the inputs, use the default overload or "
|
||||
f"clone them manually beforehand."
|
||||
)
|
||||
|
||||
if arg.op == "placeholder":
|
||||
# Graph input that maybe_inplace might modify.
|
||||
# Mark it so downstream passes know it's donated.
|
||||
# TODO(luka) store in placeholder node meta once supported
|
||||
pass_context.donated_input_ids.add(node_to_idx[arg])
|
||||
|
||||
# Same signature, just replace the overload that's called.
|
||||
node.target = ir_op.torch_op
|
||||
self.functionalized_ops[ir_op.name] += 1
|
||||
|
||||
count = sum(self.functionalized_ops.values())
|
||||
ops = ",".join(self.functionalized_ops.keys())
|
||||
logger.debug("Donated input IDs: %s", pass_context.donated_input_ids)
|
||||
logger.debug(
|
||||
"%s functionalized %d vLLM IR nodes for op(s) %s",
|
||||
self.pass_name,
|
||||
count,
|
||||
ops,
|
||||
)
|
||||
@@ -10,7 +10,6 @@ from torch._inductor.pattern_matcher import (
|
||||
PatternMatcherPass,
|
||||
register_graph_pattern,
|
||||
)
|
||||
from torch._ops import OpOverload, OpOverloadPacket
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.ir.op import IrOp
|
||||
@@ -18,41 +17,11 @@ from vllm.logger import init_logger
|
||||
from vllm.logging_utils import lazy
|
||||
|
||||
from ..vllm_inductor_pass import VllmInductorPass
|
||||
from .utils import get_ir_op
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_default_overload(op: OpOverload | OpOverloadPacket) -> OpOverload:
|
||||
if isinstance(op, OpOverloadPacket):
|
||||
return op.default
|
||||
assert isinstance(op, OpOverload), "Expected an OpOverload or OpOverloadPacket"
|
||||
return op
|
||||
|
||||
|
||||
def get_ir_op(node: fx.Node) -> IrOp | None:
|
||||
if node.op != "call_function":
|
||||
return None
|
||||
|
||||
if not isinstance(node.target, (OpOverload, OpOverloadPacket)):
|
||||
return None
|
||||
|
||||
op_overload = get_default_overload(node.target)
|
||||
if op_overload.namespace != "vllm_ir":
|
||||
return None
|
||||
|
||||
op_name = op_overload._opname
|
||||
if op_name not in IrOp.registry:
|
||||
logger.warning(
|
||||
"Unknown vLLM IR op %s, there's likely an issue with torch registration, "
|
||||
"or a torch custom op was registered in the vllm_ir namespace by mistake.",
|
||||
op_name,
|
||||
)
|
||||
return None
|
||||
|
||||
ir_op = IrOp.registry[op_name]
|
||||
return ir_op
|
||||
|
||||
|
||||
class VllmIRLoweringPass(VllmInductorPass):
|
||||
"""
|
||||
This pass lowers vLLM IR ops to their implementations the priority list.
|
||||
@@ -76,7 +45,7 @@ class VllmIRLoweringPass(VllmInductorPass):
|
||||
|
||||
assert len(match.nodes) == 1, "Expected single node match"
|
||||
node = match.nodes[0]
|
||||
ir_op = get_ir_op(node)
|
||||
ir_op = get_ir_op(node) # TODO is node.target always an overload?
|
||||
assert ir_op is not None, "Expected vLLM IR op"
|
||||
assert not node.kwargs # I think there should never be kwargs here
|
||||
|
||||
@@ -86,13 +55,18 @@ class VllmIRLoweringPass(VllmInductorPass):
|
||||
self.selected_impls[ir_op.name][node.name] = ir_op_impl.provider
|
||||
|
||||
# replace_by_example wants node args, not the fake tensors
|
||||
# use func_impl_fn to properly handle in-place implementations
|
||||
# TODO(luka): Use aot_export_module to get functionalized graph
|
||||
# TODO(luka): Cache the fx_replacement to avoid re-tracing the same impl
|
||||
|
||||
# Defaults not present on node.args but required for replacement tracing
|
||||
bound_args = ir_op._py_signature.bind(*node.args)
|
||||
bound_args.apply_defaults()
|
||||
match.replace_by_example(ir_op_impl.impl_fn, bound_args.args)
|
||||
# It is not safe to run functional passes (like DCE) on the replacements
|
||||
# as they might not be functional.
|
||||
match.replace_by_example(
|
||||
ir_op_impl.func_impl_fn, bound_args.args, run_functional_passes=False
|
||||
)
|
||||
|
||||
@VllmInductorPass.time_and_log
|
||||
def __call__(self, graph: fx.Graph) -> None:
|
||||
@@ -136,7 +110,7 @@ class VllmIRLoweringPass(VllmInductorPass):
|
||||
|
||||
if failed_nodes or failed_ops:
|
||||
logger.warning("Failed to lower vLLM IR ops: %s", ",".join(failed_ops))
|
||||
logger.warning("Full node list: %s", failed_nodes)
|
||||
logger.warning("Full node list: %s", ",".join(str(n) for n in failed_nodes))
|
||||
|
||||
def uuid(self) -> str:
|
||||
"""
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from torch import fx
|
||||
from torch._ops import OpOverload, OpOverloadPacket
|
||||
|
||||
from vllm.ir.op import IrOp
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def overload_or_default(op: OpOverload | OpOverloadPacket) -> OpOverload:
|
||||
if isinstance(op, OpOverloadPacket):
|
||||
return op.default
|
||||
assert isinstance(op, OpOverload), "Expected an OpOverload or OpOverloadPacket"
|
||||
return op
|
||||
|
||||
|
||||
def get_ir_op(node: fx.Node) -> IrOp | None:
|
||||
if node.op != "call_function":
|
||||
return None
|
||||
|
||||
if not isinstance(node.target, (OpOverload, OpOverloadPacket)):
|
||||
return None
|
||||
|
||||
op_overload = overload_or_default(node.target)
|
||||
if op_overload.namespace != "vllm_ir":
|
||||
return None
|
||||
|
||||
op_name = op_overload._opname
|
||||
if op_name not in IrOp.registry:
|
||||
logger.warning(
|
||||
"Unknown vLLM IR op %s, there's likely an issue with torch registration, "
|
||||
"or a torch custom op was registered in the vllm_ir namespace by mistake.",
|
||||
op_name,
|
||||
)
|
||||
return None
|
||||
|
||||
ir_op = IrOp.registry[op_name]
|
||||
return ir_op
|
||||
@@ -14,6 +14,7 @@ from vllm.logger import init_logger
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.system_utils import set_env_var
|
||||
|
||||
from .ir.clone_elimination import UnsafeCloneEliminationPass
|
||||
from .ir.lowering_pass import VllmIRLoweringPass
|
||||
from .vllm_inductor_pass import VllmInductorPass, VllmPatternMatcherPass
|
||||
|
||||
@@ -115,6 +116,8 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
# DCE handles mutating ops correctly as well.
|
||||
self.ir_lowering(graph)
|
||||
VllmInductorPass.dump_prefix += 1
|
||||
self.clone_elimination(graph)
|
||||
VllmInductorPass.dump_prefix += 1
|
||||
|
||||
# clean up after lowering again
|
||||
self.post_cleanup(graph)
|
||||
@@ -149,11 +152,12 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [MiniMaxQKNormPass(config)]
|
||||
|
||||
if self.pass_config.fuse_norm_quant:
|
||||
self.passes += [RMSNormQuantFusionPass(config)]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
self.passes += [
|
||||
RocmAiterRMSNormQuantFusionPass(config),
|
||||
]
|
||||
self.passes += [RMSNormQuantFusionPass(config)]
|
||||
|
||||
if self.pass_config.fuse_act_quant:
|
||||
self.passes += [ActivationQuantFusionPass(config)]
|
||||
if rocm_aiter_ops.is_enabled():
|
||||
@@ -179,6 +183,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
self.passes += [QKNormRoPEFusionPass(config)]
|
||||
|
||||
self.ir_lowering = VllmIRLoweringPass(config)
|
||||
self.clone_elimination = UnsafeCloneEliminationPass(config)
|
||||
self.post_cleanup = PostCleanupPass(config)
|
||||
self.fix_functionalization = FixFunctionalizationPass(config)
|
||||
|
||||
@@ -200,6 +205,7 @@ class PostGradPassManager(CustomGraphPass): # type: ignore[misc]
|
||||
|
||||
passes.append(self.post_cleanup.uuid())
|
||||
passes.append(self.ir_lowering.uuid())
|
||||
passes.append(self.clone_elimination.uuid())
|
||||
passes.append(self.post_cleanup.uuid())
|
||||
passes.append(self.fix_functionalization.uuid())
|
||||
|
||||
|
||||
Reference in New Issue
Block a user