Fix Transformers modelling backend RMSNormFuser.fuse performance (#52766)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -60,6 +60,54 @@ class LayerNorm(RMSNorm):
|
||||
return self.weight * self._rms(x)
|
||||
|
||||
|
||||
class NamedEpsRMSNorm(nn.Module):
|
||||
"""Holds eps under `attr`, with no `variance_epsilon` to find instead."""
|
||||
|
||||
attr = "eps"
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
setattr(self, self.attr, eps)
|
||||
|
||||
def forward(self, x):
|
||||
eps = getattr(self, self.attr)
|
||||
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps))
|
||||
|
||||
|
||||
class ToleranceRMSNorm(NamedEpsRMSNorm):
|
||||
"""Names eps something other than `eps`, so only its value identifies it."""
|
||||
|
||||
attr = "tolerance"
|
||||
|
||||
|
||||
class LiteralEpsRMSNorm(RMSNorm):
|
||||
"""eps is a literal in the source, and an unrelated attribute happens to
|
||||
hold the same value: matching on the value alone would bind to it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-5):
|
||||
super().__init__(hidden, eps)
|
||||
self.epsilon_lookalike = 1e-4
|
||||
|
||||
def forward(self, x):
|
||||
return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4))
|
||||
|
||||
|
||||
class AmbiguousEpsRMSNorm(nn.Module):
|
||||
"""Two attributes hold the eps value, and the forward reads the second, so
|
||||
their order cannot pick it."""
|
||||
|
||||
def __init__(self, hidden: int = 16, eps: float = 1e-6):
|
||||
super().__init__()
|
||||
self.weight = nn.Parameter(torch.ones(hidden))
|
||||
self.decoy_eps = eps
|
||||
self.variance_epsilon = eps
|
||||
|
||||
def forward(self, x):
|
||||
var = x.pow(2).mean(-1, True)
|
||||
return self.weight * (x * torch.rsqrt(var + self.variance_epsilon))
|
||||
|
||||
|
||||
class NotAnRMSNorm(RMSNorm):
|
||||
"""Mean-subtracting LayerNorm-like math -> not an RMSNorm."""
|
||||
|
||||
@@ -207,6 +255,48 @@ def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm])
|
||||
def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config):
|
||||
"""The eps attribute is identified by holding the traced value, so a norm
|
||||
stays per-instance correct whatever it names it."""
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = cls(16, eps=eps)
|
||||
fuser = get_fuser(module)
|
||||
assert fuser.eps_attr == cls.attr
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog):
|
||||
"""A literal eps is recognised as coming from no attribute, even when one
|
||||
holds the same value, and is taken from the traced source instead."""
|
||||
logger = "vllm.model_executor.models.transformers.fusers.rms_norm"
|
||||
with caplog.at_level("DEBUG", logger=logger), torch.device("meta"):
|
||||
module = LiteralEpsRMSNorm()
|
||||
fuser = get_fuser(module)
|
||||
built = fuser.fuse(module, "norm", default_vllm_config)
|
||||
assert fuser.eps_attr is None
|
||||
assert built.variance_epsilon == 1e-4
|
||||
assert "does not hold its eps" in caplog.text
|
||||
|
||||
|
||||
def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config):
|
||||
"""When several attributes hold the eps value, the one the forward actually
|
||||
reads is identified, and they are all left as they were found."""
|
||||
with torch.device("meta"):
|
||||
module = AmbiguousEpsRMSNorm(16, eps=1e-6)
|
||||
before = dict(vars(module))
|
||||
fuser = get_fuser(module)
|
||||
assert fuser.eps_attr == "variance_epsilon"
|
||||
assert vars(module) == before
|
||||
|
||||
other = AmbiguousEpsRMSNorm(16, eps=1e-5)
|
||||
other.decoy_eps = 1e-3
|
||||
built = fuser.fuse(other, "norm", default_vllm_config)
|
||||
assert built.variance_epsilon == 1e-5
|
||||
|
||||
|
||||
def test_fused_norm_is_gather_capable(default_vllm_config):
|
||||
"""Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded
|
||||
projection (OLMoE-style) self-corrects at runtime with no QKV-specific
|
||||
|
||||
@@ -22,7 +22,6 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
single_self_call,
|
||||
trace,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
@@ -57,8 +56,10 @@ def _norm_size(norm: nn.Module) -> int:
|
||||
|
||||
def _is_rms_norm(module: nn.Module) -> bool:
|
||||
"""Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher."""
|
||||
graph = trace(module)
|
||||
return graph is not None and RMSNormFuser.match(graph, module) is not None
|
||||
# Go via `get_fuser` because it caches
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
|
||||
return isinstance(get_fuser(module), RMSNormFuser)
|
||||
|
||||
|
||||
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
|
||||
|
||||
@@ -15,6 +15,7 @@ from vllm.distributed import (
|
||||
)
|
||||
from vllm.distributed.parallel_state import model_parallel_is_initialized
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
@@ -32,6 +33,8 @@ from vllm.model_executor.models.transformers.layers import (
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _is_squared(node: object, x: fx.Node) -> bool:
|
||||
"""`x**2`, `x.square()` or `x * x`, through any dtype casts."""
|
||||
@@ -122,6 +125,10 @@ class RMSNormFuser(BaseFuser):
|
||||
"""Gemma-style `(1 + weight)` scaling (weight initialised at zero)."""
|
||||
source_cls: str
|
||||
"""Class name of the norm this was matched from (for logging)."""
|
||||
eps_attr: str | None = None
|
||||
"""Attribute holding `eps`, read per instance in `fuse`."""
|
||||
eps: float | None = None
|
||||
"""`eps` itself, when it is not held in an attribute (see `_eps_source`)."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm"
|
||||
@@ -140,7 +147,13 @@ class RMSNormFuser(BaseFuser):
|
||||
if rms_norm is not None and rms_norm.args and peel(rms_norm.args[0]) is x:
|
||||
if _has_trailing_compute(graph, rms_norm):
|
||||
return None
|
||||
return cls(zero_centered=False, source_cls=type(module).__name__)
|
||||
eps_attr, eps = cls._eps_source(graph, module)
|
||||
return cls(
|
||||
zero_centered=False,
|
||||
source_cls=type(module).__name__,
|
||||
eps_attr=eps_attr,
|
||||
eps=eps,
|
||||
)
|
||||
# Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern.
|
||||
# The rsqrt over the mean-square variance is the spine of the norm.
|
||||
rsqrt = None
|
||||
@@ -169,7 +182,51 @@ class RMSNormFuser(BaseFuser):
|
||||
# The norm must be the last compute in forward, or it is not a pure norm.
|
||||
if _has_trailing_compute(graph, tail):
|
||||
return None
|
||||
return cls(zero_centered=zero_centered, source_cls=type(module).__name__)
|
||||
eps_attr, eps = cls._eps_source(graph, module)
|
||||
return cls(
|
||||
zero_centered=zero_centered,
|
||||
source_cls=type(module).__name__,
|
||||
eps_attr=eps_attr,
|
||||
eps=eps,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _eps_source(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[str | None, float | None]:
|
||||
"""Where `fuse` should read `eps` from, resolved once per class."""
|
||||
eps = cls._eps_from_graph(graph)
|
||||
if eps is None:
|
||||
return None, None
|
||||
# Whatever supplied the constant must still equal it.
|
||||
candidates = {
|
||||
name: value
|
||||
for name, value in vars(module).items()
|
||||
if isinstance(value, float) and value == eps
|
||||
}
|
||||
# Use unique markers and retrace to verify exactly which attribute is eps.
|
||||
markers = {float(-index - 1): name for index, name in enumerate(candidates)}
|
||||
marked = None
|
||||
if markers:
|
||||
try:
|
||||
for marker, name in markers.items():
|
||||
setattr(module, name, marker)
|
||||
if (remarked := trace(module)) is not None:
|
||||
marked = cls._eps_from_graph(remarked)
|
||||
finally:
|
||||
for name, value in candidates.items():
|
||||
setattr(module, name, value)
|
||||
if (name := markers.get(marked)) is not None:
|
||||
return name, None
|
||||
logger.debug_once(
|
||||
"%s does not hold its eps (%s) in an attribute. Every instance in this "
|
||||
"model will use the value traced from this instance. If this is not "
|
||||
"desired, consider storing and reading eps using attribute of %s.",
|
||||
type(module).__name__,
|
||||
eps,
|
||||
type(module).__name__,
|
||||
)
|
||||
return None, eps
|
||||
|
||||
@staticmethod
|
||||
def _eps_from_graph(graph: fx.Graph) -> float | None:
|
||||
@@ -180,7 +237,7 @@ class RMSNormFuser(BaseFuser):
|
||||
if fused is not None and fused.args and peel(fused.args[0]) is x:
|
||||
args, kwargs = fused.args, fused.kwargs
|
||||
eps = args[3] if len(args) > 3 else kwargs.get("eps")
|
||||
return eps if isinstance(eps, (int, float)) else None
|
||||
return float(eps) if isinstance(eps, (int, float)) else None
|
||||
for node in graph.nodes:
|
||||
if is_op(node, "rsqrt") and (eps := _variance_eps(node, x)) is not None:
|
||||
return eps
|
||||
@@ -196,10 +253,9 @@ class RMSNormFuser(BaseFuser):
|
||||
weight = getattr(module, "weight", None)
|
||||
has_weight = weight is not None
|
||||
hidden_size = weight.size(0) if has_weight else 0
|
||||
graph = trace(module)
|
||||
eps = self._eps_from_graph(graph) if graph is not None else None
|
||||
if eps is None:
|
||||
# If eps not in graph, match torch behaviour.
|
||||
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
|
||||
if not isinstance(eps, (int, float)):
|
||||
# If eps was not detected, match torch behaviour.
|
||||
dtype = weight.dtype if has_weight else vllm_config.model_config.dtype
|
||||
eps = torch.finfo(dtype).eps
|
||||
if self.zero_centered:
|
||||
|
||||
Reference in New Issue
Block a user