[vLLM IR] rework gemma_rms_norm (#39014)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Jiangyun Zhu <riverclouds.zhu@qq.com> Co-authored-by: Luka Govedič <ProExpertProg@users.noreply.github.com>
This commit is contained in:
co-authored by
Luka Govedič
parent
da4c0e4db9
commit
8060bb0333
@@ -12,6 +12,9 @@ from torch._higher_order_ops.auto_functionalize import auto_functionalized
|
||||
from torch._inductor.pattern_matcher import PatternMatcherPass
|
||||
|
||||
import vllm.ir.ops
|
||||
from vllm.compilation.passes.fusion.rms_quant_fusion import (
|
||||
_rms_input_weight_dtype_match,
|
||||
)
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.utils import Range
|
||||
from vllm.distributed import get_tp_group, tensor_model_parallel_all_reduce
|
||||
@@ -320,7 +323,12 @@ class AllReduceRMSNormPattern(BasePattern):
|
||||
return allreduce[3], allreduce[1]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.get_inputs(), pm.fwd_only, pm_pass
|
||||
pattern,
|
||||
replacement,
|
||||
self.get_inputs(),
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
@@ -459,7 +467,12 @@ class AllReduceFusedRMSNormStaticQuantFP8Pattern(BasePattern):
|
||||
return allreduce[4], allreduce[1]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.get_inputs(), pm.fwd_only, pm_pass
|
||||
pattern,
|
||||
replacement,
|
||||
self.get_inputs(),
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
@@ -621,7 +634,12 @@ class AllReduceFusedRMSNormStaticQuantNVFP4Pattern(BasePattern):
|
||||
return allreduce[4], allreduce[1], allreduce[5]
|
||||
|
||||
pm.register_replacement(
|
||||
pattern, replacement, self.get_inputs(), pm.fwd_only, pm_pass
|
||||
pattern,
|
||||
replacement,
|
||||
self.get_inputs(),
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -38,6 +38,22 @@ FP8_DTYPE = current_platform.fp8_dtype()
|
||||
FP4_DTYPE = torch.uint8
|
||||
|
||||
|
||||
_RMS_NORM_OP = torch.ops.vllm_ir.rms_norm.default
|
||||
|
||||
|
||||
# TODO: extend rmsnorm quant kernels to support mixed input/weight dtypes,
|
||||
# and remove this check.
|
||||
def _rms_input_weight_dtype_match(match: pm.Match) -> bool:
|
||||
"""Prevent fusion when rms_norm input and weight dtypes differ."""
|
||||
for node in match.nodes:
|
||||
if node.target == _RMS_NORM_OP:
|
||||
# rms_norm(x, weight, epsilon, variance_size)
|
||||
x, weight = node.args[0], node.args[1]
|
||||
if isinstance(x, fx.Node) and isinstance(weight, fx.Node):
|
||||
return x.meta["val"].dtype == weight.meta["val"].dtype
|
||||
return True
|
||||
|
||||
|
||||
def empty_bf16(*args: Any, **kwargs: Any) -> torch.Tensor:
|
||||
return torch.empty(*args, **kwargs, dtype=torch.bfloat16, device="cuda")
|
||||
|
||||
@@ -186,7 +202,14 @@ class RMSNormStaticQuantPattern(RMSNormQuantPattern):
|
||||
]
|
||||
pattern(*inputs)
|
||||
|
||||
pm.register_replacement(pattern, replacement, inputs, pm.fwd_only, pm_pass)
|
||||
pm.register_replacement(
|
||||
pattern,
|
||||
replacement,
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
|
||||
@@ -249,6 +272,7 @@ class FusedAddRMSNormStaticQuantPattern(RMSNormQuantPattern):
|
||||
inputs,
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
@@ -350,6 +374,7 @@ class FusedAddRMSNormGroupQuantPattern(RMSNormQuantPattern):
|
||||
self.rmsnorm_matcher.inputs() + [scale],
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
@@ -445,6 +470,7 @@ class RMSNormGroupQuantPattern(RMSNormQuantPattern):
|
||||
],
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
@@ -503,6 +529,7 @@ class RMSNormDynamicQuantPattern(RMSNormQuantPattern):
|
||||
],
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
@@ -559,6 +586,7 @@ class FusedAddRMSNormDynamicQuantPattern(RMSNormQuantPattern):
|
||||
self.rmsnorm_matcher.inputs(),
|
||||
pm.fwd_only,
|
||||
pm_pass,
|
||||
extra_check=_rms_input_weight_dtype_match,
|
||||
)
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user