diff --git a/tests/models/transformers/fusers/test_rms_norm.py b/tests/models/transformers/fusers/test_rms_norm.py index 711f9b8b22..ac259b66b4 100644 --- a/tests/models/transformers/fusers/test_rms_norm.py +++ b/tests/models/transformers/fusers/test_rms_norm.py @@ -60,6 +60,54 @@ class LayerNorm(RMSNorm): return self.weight * self._rms(x) +class NamedEpsRMSNorm(nn.Module): + """Holds eps under `attr`, with no `variance_epsilon` to find instead.""" + + attr = "eps" + + def __init__(self, hidden: int = 16, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden)) + setattr(self, self.attr, eps) + + def forward(self, x): + eps = getattr(self, self.attr) + return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + eps)) + + +class ToleranceRMSNorm(NamedEpsRMSNorm): + """Names eps something other than `eps`, so only its value identifies it.""" + + attr = "tolerance" + + +class LiteralEpsRMSNorm(RMSNorm): + """eps is a literal in the source, and an unrelated attribute happens to + hold the same value: matching on the value alone would bind to it.""" + + def __init__(self, hidden: int = 16, eps: float = 1e-5): + super().__init__(hidden, eps) + self.epsilon_lookalike = 1e-4 + + def forward(self, x): + return self.weight * (x * torch.rsqrt(x.pow(2).mean(-1, True) + 1e-4)) + + +class AmbiguousEpsRMSNorm(nn.Module): + """Two attributes hold the eps value, and the forward reads the second, so + their order cannot pick it.""" + + def __init__(self, hidden: int = 16, eps: float = 1e-6): + super().__init__() + self.weight = nn.Parameter(torch.ones(hidden)) + self.decoy_eps = eps + self.variance_epsilon = eps + + def forward(self, x): + var = x.pow(2).mean(-1, True) + return self.weight * (x * torch.rsqrt(var + self.variance_epsilon)) + + class NotAnRMSNorm(RMSNorm): """Mean-subtracting LayerNorm-like math -> not an RMSNorm.""" @@ -207,6 +255,48 @@ def test_eps_is_derived_per_instance(default_vllm_config): assert built.variance_epsilon == eps +@pytest.mark.parametrize("cls", [NamedEpsRMSNorm, ToleranceRMSNorm]) +def test_eps_attr_is_found_by_value_not_name(cls, default_vllm_config): + """The eps attribute is identified by holding the traced value, so a norm + stays per-instance correct whatever it names it.""" + with torch.device("meta"): + for eps in (1e-5, 1e-6): + module = cls(16, eps=eps) + fuser = get_fuser(module) + assert fuser.eps_attr == cls.attr + built = fuser.fuse(module, "norm", default_vllm_config) + assert built.variance_epsilon == eps + + +def test_literal_eps_is_not_mistaken_for_an_attribute(default_vllm_config, caplog): + """A literal eps is recognised as coming from no attribute, even when one + holds the same value, and is taken from the traced source instead.""" + logger = "vllm.model_executor.models.transformers.fusers.rms_norm" + with caplog.at_level("DEBUG", logger=logger), torch.device("meta"): + module = LiteralEpsRMSNorm() + fuser = get_fuser(module) + built = fuser.fuse(module, "norm", default_vllm_config) + assert fuser.eps_attr is None + assert built.variance_epsilon == 1e-4 + assert "does not hold its eps" in caplog.text + + +def test_ambiguous_eps_attrs_are_disambiguated(default_vllm_config): + """When several attributes hold the eps value, the one the forward actually + reads is identified, and they are all left as they were found.""" + with torch.device("meta"): + module = AmbiguousEpsRMSNorm(16, eps=1e-6) + before = dict(vars(module)) + fuser = get_fuser(module) + assert fuser.eps_attr == "variance_epsilon" + assert vars(module) == before + + other = AmbiguousEpsRMSNorm(16, eps=1e-5) + other.decoy_eps = 1e-3 + built = fuser.fuse(other, "norm", default_vllm_config) + assert built.variance_epsilon == 1e-5 + + def test_fused_norm_is_gather_capable(default_vllm_config): """Every weighted fused norm is emitted gather-capable, so a norm on a head-sharded projection (OLMoE-style) self-corrects at runtime with no QKV-specific diff --git a/vllm/model_executor/models/transformers/fusers/mla.py b/vllm/model_executor/models/transformers/fusers/mla.py index 8e17bff490..db29ded911 100644 --- a/vllm/model_executor/models/transformers/fusers/mla.py +++ b/vllm/model_executor/models/transformers/fusers/mla.py @@ -22,7 +22,6 @@ from vllm.model_executor.models.transformers.fx_utils import ( replace_expr, returned_linear, single_self_call, - trace, upstream_linear, ) from vllm.model_executor.models.transformers.utils import ( @@ -57,8 +56,10 @@ def _norm_size(norm: nn.Module) -> int: def _is_rms_norm(module: nn.Module) -> bool: """Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher.""" - graph = trace(module) - return graph is not None and RMSNormFuser.match(graph, module) is not None + # Go via `get_fuser` because it caches + from vllm.model_executor.models.transformers.fuser import get_fuser + + return isinstance(get_fuser(module), RMSNormFuser) def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int: diff --git a/vllm/model_executor/models/transformers/fusers/rms_norm.py b/vllm/model_executor/models/transformers/fusers/rms_norm.py index 7241f8539c..47d20085a2 100644 --- a/vllm/model_executor/models/transformers/fusers/rms_norm.py +++ b/vllm/model_executor/models/transformers/fusers/rms_norm.py @@ -15,6 +15,7 @@ from vllm.distributed import ( ) from vllm.distributed.parallel_state import model_parallel_is_initialized from vllm.distributed.utils import split_tensor_along_last_dim +from vllm.logger import init_logger from vllm.model_executor.models.transformers.fusers.base import BaseFuser from vllm.model_executor.models.transformers.fx_utils import ( find_node, @@ -32,6 +33,8 @@ from vllm.model_executor.models.transformers.layers import ( if TYPE_CHECKING: from vllm.config import VllmConfig +logger = init_logger(__name__) + def _is_squared(node: object, x: fx.Node) -> bool: """`x**2`, `x.square()` or `x * x`, through any dtype casts.""" @@ -122,6 +125,10 @@ class RMSNormFuser(BaseFuser): """Gemma-style `(1 + weight)` scaling (weight initialised at zero).""" source_cls: str """Class name of the norm this was matched from (for logging).""" + eps_attr: str | None = None + """Attribute holding `eps`, read per instance in `fuse`.""" + eps: float | None = None + """`eps` itself, when it is not held in an attribute (see `_eps_source`).""" def info(self, name: str) -> str: norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm" @@ -140,7 +147,13 @@ class RMSNormFuser(BaseFuser): if rms_norm is not None and rms_norm.args and peel(rms_norm.args[0]) is x: if _has_trailing_compute(graph, rms_norm): return None - return cls(zero_centered=False, source_cls=type(module).__name__) + eps_attr, eps = cls._eps_source(graph, module) + return cls( + zero_centered=False, + source_cls=type(module).__name__, + eps_attr=eps_attr, + eps=eps, + ) # Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern. # The rsqrt over the mean-square variance is the spine of the norm. rsqrt = None @@ -169,7 +182,51 @@ class RMSNormFuser(BaseFuser): # The norm must be the last compute in forward, or it is not a pure norm. if _has_trailing_compute(graph, tail): return None - return cls(zero_centered=zero_centered, source_cls=type(module).__name__) + eps_attr, eps = cls._eps_source(graph, module) + return cls( + zero_centered=zero_centered, + source_cls=type(module).__name__, + eps_attr=eps_attr, + eps=eps, + ) + + @classmethod + def _eps_source( + cls, graph: fx.Graph, module: nn.Module + ) -> tuple[str | None, float | None]: + """Where `fuse` should read `eps` from, resolved once per class.""" + eps = cls._eps_from_graph(graph) + if eps is None: + return None, None + # Whatever supplied the constant must still equal it. + candidates = { + name: value + for name, value in vars(module).items() + if isinstance(value, float) and value == eps + } + # Use unique markers and retrace to verify exactly which attribute is eps. + markers = {float(-index - 1): name for index, name in enumerate(candidates)} + marked = None + if markers: + try: + for marker, name in markers.items(): + setattr(module, name, marker) + if (remarked := trace(module)) is not None: + marked = cls._eps_from_graph(remarked) + finally: + for name, value in candidates.items(): + setattr(module, name, value) + if (name := markers.get(marked)) is not None: + return name, None + logger.debug_once( + "%s does not hold its eps (%s) in an attribute. Every instance in this " + "model will use the value traced from this instance. If this is not " + "desired, consider storing and reading eps using attribute of %s.", + type(module).__name__, + eps, + type(module).__name__, + ) + return None, eps @staticmethod def _eps_from_graph(graph: fx.Graph) -> float | None: @@ -180,7 +237,7 @@ class RMSNormFuser(BaseFuser): if fused is not None and fused.args and peel(fused.args[0]) is x: args, kwargs = fused.args, fused.kwargs eps = args[3] if len(args) > 3 else kwargs.get("eps") - return eps if isinstance(eps, (int, float)) else None + return float(eps) if isinstance(eps, (int, float)) else None for node in graph.nodes: if is_op(node, "rsqrt") and (eps := _variance_eps(node, x)) is not None: return eps @@ -196,10 +253,9 @@ class RMSNormFuser(BaseFuser): weight = getattr(module, "weight", None) has_weight = weight is not None hidden_size = weight.size(0) if has_weight else 0 - graph = trace(module) - eps = self._eps_from_graph(graph) if graph is not None else None - if eps is None: - # If eps not in graph, match torch behaviour. + eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps + if not isinstance(eps, (int, float)): + # If eps was not detected, match torch behaviour. dtype = weight.dtype if has_weight else vllm_config.model_config.dtype eps = torch.finfo(dtype).eps if self.zero_centered: