diff --git a/tests/models/transformers/fusers/test_linear.py b/tests/models/transformers/fusers/test_linear.py index 546dafb792..eff78fa290 100644 --- a/tests/models/transformers/fusers/test_linear.py +++ b/tests/models/transformers/fusers/test_linear.py @@ -368,12 +368,12 @@ def test_qkv_identifies_output_projection(): assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj" -def test_fuser_is_cached_per_class(): +def test_fuser_is_cached_per_class_and_structure(): with torch.device("meta"): fuser_a = get_fuser(GLUMLP()) fuser_b = get_fuser(GLUMLP()) assert fuser_a is fuser_b - assert GLUMLP in get_fuser.cache + assert any(key[0] is GLUMLP for key in get_fuser.cache) @pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP]) @@ -460,8 +460,7 @@ def test_unfusable_modules_are_not_fused(cls, default_vllm_config): fuser = get_fuser(module) # Either no pattern matches the class, or this instance fails validation # (`recursive_replace` gates fusion and its weight mappings on `validate`) - model_config = default_vllm_config.model_config - assert fuser is None or not fuser.validate(module, model_config) + assert fuser is None or not fuser.validate(module, default_vllm_config) def test_act_and_mul_derived_from_module(default_vllm_config): diff --git a/tests/models/transformers/fusers/test_rms_norm.py b/tests/models/transformers/fusers/test_rms_norm.py index 6497b98c4b..131897b192 100644 --- a/tests/models/transformers/fusers/test_rms_norm.py +++ b/tests/models/transformers/fusers/test_rms_norm.py @@ -148,11 +148,13 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c # `default_vllm_config` supplies the config context the CustomOp needs; the # weightless path reads hidden size from the model config, so stub it. - model_config = SimpleNamespace(get_hidden_size=lambda: 16) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace(get_hidden_size=lambda: 16) + ) with torch.device("meta"): module = cls() fuser = get_fuser(module) - built = fuser.fuse(module, "norm", model_config, None) + built = fuser.fuse(module, "norm", vllm_config) from vllm.model_executor.models.transformers.fusers.rms_norm import ( TPAwareNormMixin, ) @@ -176,8 +178,9 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config): fuser = get_fuser(module) assert isinstance(fuser, RMSNormFuser) assert not fuser.zero_centered - model_config = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32) - built = fuser.fuse(module, "norm", model_config, None) + mc = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32) + vllm_config = SimpleNamespace(model_config=mc) + built = fuser.fuse(module, "norm", vllm_config) assert isinstance(built, VLLMRMSNorm) assert built.variance_epsilon == torch.finfo(torch.float32).eps @@ -185,11 +188,13 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config): def test_eps_is_derived_per_instance(default_vllm_config): """Two instances of the same norm class with different eps must fuse to their own eps: the type-cached fuser holds only structure, not this value.""" - model_config = SimpleNamespace(get_hidden_size=lambda: 16) + vllm_config = SimpleNamespace( + model_config=SimpleNamespace(get_hidden_size=lambda: 16) + ) with torch.device("meta"): for eps in (1e-5, 1e-6): module = RMSNorm(16, eps=eps) - built = get_fuser(module).fuse(module, "norm", model_config, None) + built = get_fuser(module).fuse(module, "norm", vllm_config) assert built.variance_epsilon == eps diff --git a/vllm/model_executor/models/transformers/base.py b/vllm/model_executor/models/transformers/base.py index 24cad1da20..35e5c9d1a0 100644 --- a/vllm/model_executor/models/transformers/base.py +++ b/vllm/model_executor/models/transformers/base.py @@ -104,6 +104,7 @@ class Base( super().__init__() logger.info("Using Transformers modeling backend.") + self.vllm_config = vllm_config self.config = vllm_config.model_config.hf_config self.text_config = self.config.get_text_config() self.cache_config = vllm_config.cache_config @@ -357,7 +358,7 @@ class Base( tip = get_feature_request_tip( self.model_config.model, self.model_config.trust_remote_code ) - logger.warning( + logger.warning_once( "%s does not define a pipeline parallel plan. The Transformers " "modeling backend will infer the split from the layers of %s in order " "of declaration and keep parameter-free modules on every rank. This " @@ -452,7 +453,7 @@ class Base( # Prefix the patterns because we always start from `self.model` tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()} # Detect fusable patterns once per module class (cached, so this is cheap) - fusers = Fusers(self.model, self.model_config) + fusers = Fusers(self.model, self.vllm_config) def register_fusion(fuser: BaseFuser, prefix: str): """Register a fused layer's mappings just before it is built.""" @@ -503,9 +504,7 @@ class Base( new_module = replace_conv_class(child_module) elif (fuser := fusers[child_module]) is not None: register_fusion(fuser, qual_name) - new_module = fuser.fuse( - child_module, qual_name, self.model_config, self.quant_config - ) + new_module = fuser.fuse(child_module, qual_name, self.vllm_config) logger.info_once(fuser.info(child_name)) _recursive_replace(new_module, prefix=qual_name) elif not isinstance(child_module, MoERunner): diff --git a/vllm/model_executor/models/transformers/fuser.py b/vllm/model_executor/models/transformers/fuser.py index 0aa7c419ce..13d84a245d 100644 --- a/vllm/model_executor/models/transformers/fuser.py +++ b/vllm/model_executor/models/transformers/fuser.py @@ -25,14 +25,19 @@ from vllm.model_executor.models.transformers.fusers import ( from vllm.model_executor.models.transformers.fx_utils import trace if TYPE_CHECKING: - from vllm.config.model import ModelConfig + from vllm.config import VllmConfig logger = init_logger(__name__) -@cached(cache={}, key=type) +def key(module: nn.Module) -> tuple: + """Cache key for `get_fuser`. Considers module type and its immediate children.""" + return (type(module), tuple(name for name, _ in module.named_children())) + + +@cached(cache={}, key=key) def get_fuser(module: nn.Module) -> BaseFuser | None: - """The fuser for `type(module)` (cached per class), or `None` if no match.""" + """The fuser for `module`'s class and shape (cached), or `None` if no match.""" # Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a # leaf module (raw tensor math, no submodules). Nothing else can match, and # tracing is skipped for it. @@ -48,11 +53,14 @@ def get_fuser(module: nn.Module) -> BaseFuser | None: try: fuser.update_forward(module) except Exception as exc: - # An unrecognised source just means we cannot fuse here. logger.debug( - "Could not rewrite %s for fusion: %s", type(module), exc + "Attempted to fuse %s using %s but failed " + "to update its forward method: %s", + type(module), + fuser_cls.__name__, + exc, ) - return None + continue return fuser # A norm we could not match structurally is left unfused; flag likely misses. if module.__class__.__name__.endswith("RMSNorm"): @@ -67,12 +75,12 @@ def get_fuser(module: nn.Module) -> BaseFuser | None: class Fusers(UserDict): """Mapping from module class to fuser, for all fusable classes in a model.""" - def __init__(self, model: nn.Module, model_config: "ModelConfig"): - self.model_config = model_config + def __init__(self, model: nn.Module, vllm_config: "VllmConfig"): + self.vllm_config = vllm_config super().__init__({type(m): get_fuser(m) for m in model.modules()}) def __getitem__(self, m: nn.Module) -> BaseFuser | None: fuser = self.data.get(type(m)) - if fuser is not None and fuser.validate(m, self.model_config): + if fuser is not None and fuser.validate(m, self.vllm_config): return fuser return None diff --git a/vllm/model_executor/models/transformers/fusers/base.py b/vllm/model_executor/models/transformers/fusers/base.py index 54fb2d0916..abeff73e3e 100644 --- a/vllm/model_executor/models/transformers/fusers/base.py +++ b/vllm/model_executor/models/transformers/fusers/base.py @@ -13,8 +13,7 @@ from torch import fx, nn from vllm.model_executor.models.utils import ShardId, maybe_prefix if TYPE_CHECKING: - from vllm.config.model import ModelConfig - from vllm.model_executor.layers.quantization import QuantizationConfig + from vllm.config import VllmConfig @dataclass @@ -36,16 +35,12 @@ class BaseFuser(ABC): """Match the pattern in `graph`, returning a fuser if found.""" @abstractmethod - def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool: + def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: """Whether this fuser can be applied to this `module` instance.""" @abstractmethod def fuse( - self, - module: nn.Module, - prefix: str, - model_config: "ModelConfig", - quant_config: "QuantizationConfig", + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> nn.Module: """Apply the fusion to an already-validated `module`, returning the module to install in its place (mutated in place, or freshly built).""" @@ -123,24 +118,16 @@ class StackedFuser(BaseFuser): @abstractmethod def update_attrs( - self, - module: nn.Module, - prefix: str, - model_config: "ModelConfig", - quant_config: "QuantizationConfig", + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> None: """Replace `module`'s submodules with the merged module.""" def fuse( - self, - module: nn.Module, - prefix: str, - model_config: "ModelConfig", - quant_config: "QuantizationConfig", + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> nn.Module: """Fuse an already-validated `module` in place (see `Fusers.__getitem__`). Builds the merged submodule and binds the compiled forward.""" - self.update_attrs(module, prefix, model_config, quant_config) + self.update_attrs(module, prefix, vllm_config) module.forward = types.MethodType(self.fused_forward, module) return module diff --git a/vllm/model_executor/models/transformers/fusers/glu.py b/vllm/model_executor/models/transformers/fusers/glu.py index 951eb35777..2da84addea 100644 --- a/vllm/model_executor/models/transformers/fusers/glu.py +++ b/vllm/model_executor/models/transformers/fusers/glu.py @@ -33,8 +33,7 @@ from vllm.model_executor.models.transformers.utils import ( from vllm.model_executor.models.utils import ShardId, maybe_prefix if TYPE_CHECKING: - from vllm.config.model import ModelConfig - from vllm.model_executor.layers.quantization import QuantizationConfig + from vllm.config import VllmConfig logger = init_logger(__name__) @@ -168,7 +167,7 @@ class GLUFuser(StackedFuser): replace_expr(funcdef, muls[0], act_call) self.fused_forward = compile_forward(funcdef, fn) - def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool: + def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: act = module.get_submodule(self.act_name) if self._get_act_and_mul_name(act) is None: logger.debug("No AndMul equivalent for %s; skipping fusion", type(act)) @@ -176,12 +175,9 @@ class GLUFuser(StackedFuser): return True def update_attrs( - self, - module: nn.Module, - prefix: str, - model_config: "ModelConfig", - quant_config: "QuantizationConfig", + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> None: + quant_config = vllm_config.quant_config act_fn = self._get_act_and_mul(module.get_submodule(self.act_name)) gate = module.get_submodule(self.gate_name) up = module.get_submodule(self.up_name) diff --git a/vllm/model_executor/models/transformers/fusers/qkv.py b/vllm/model_executor/models/transformers/fusers/qkv.py index 010a0018ac..b9fe9771fe 100644 --- a/vllm/model_executor/models/transformers/fusers/qkv.py +++ b/vllm/model_executor/models/transformers/fusers/qkv.py @@ -17,6 +17,7 @@ from vllm.model_executor.models.transformers.fx_utils import ( is_linear, recover_forward, replace_expr, + returned_linear, single_self_call, ) from vllm.model_executor.models.transformers.utils import ( @@ -26,8 +27,7 @@ from vllm.model_executor.models.transformers.utils import ( from vllm.model_executor.models.utils import ShardId, maybe_prefix if TYPE_CHECKING: - from vllm.config.model import ModelConfig - from vllm.model_executor.layers.quantization import QuantizationConfig + from vllm.config import VllmConfig logger = init_logger(__name__) @@ -84,15 +84,16 @@ class QKVFuser(StackedFuser): return None q, k, v = qkv_nodes names = dict(q_name=q.target, k_name=k.target, v_name=v.target) - attn_width = module.get_submodule(q.target).out_features - candidates = [ - name - for name, child in module.named_children() - if isinstance(child, nn.Linear) - and name not in names.values() - and child.in_features == attn_width - ] - names["o_name"] = candidates[0] if len(candidates) == 1 else None + # o_proj produces the module's output. + o_name = returned_linear(graph, module) + # o_proj must be compatible with the q/k/v projections. + if o_name in names.values() or ( + o_name is not None + and module.get_submodule(o_name).in_features + != module.get_submodule(q.target).out_features + ): + o_name = None + names["o_name"] = o_name return cls(source_cls=type(module).__name__, **names) def update_forward(self, module: nn.Module) -> None: @@ -149,12 +150,12 @@ class QKVFuser(StackedFuser): replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load())) self.fused_forward = compile_forward(funcdef, fn) - def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool: + def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: """Shapes must be compatible for a single merged, head-sharded GEMM.""" q = module.get_submodule(self.q_name) k = module.get_submodule(self.k_name) v = module.get_submodule(self.v_name) - head_size = model_config.get_head_size() + head_size = vllm_config.model_config.get_head_size() compatible = ( q.in_features == k.in_features == v.in_features and len({proj.bias is None for proj in (q, k, v)}) == 1 @@ -167,13 +168,10 @@ class QKVFuser(StackedFuser): return compatible def update_attrs( - self, - module: nn.Module, - prefix: str, - model_config: "ModelConfig", - quant_config: "QuantizationConfig", + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> None: - head_size = model_config.get_head_size() + quant_config = vllm_config.quant_config + head_size = vllm_config.model_config.get_head_size() q = module.get_submodule(self.q_name) k = module.get_submodule(self.k_name) merged = QKVParallelLinear( diff --git a/vllm/model_executor/models/transformers/fusers/rms_norm.py b/vllm/model_executor/models/transformers/fusers/rms_norm.py index 829fd9a541..781b417177 100644 --- a/vllm/model_executor/models/transformers/fusers/rms_norm.py +++ b/vllm/model_executor/models/transformers/fusers/rms_norm.py @@ -21,13 +21,13 @@ from vllm.model_executor.models.transformers.fx_utils import ( find_node, forward_input_count, is_op, + output_value, peel, trace, ) if TYPE_CHECKING: - from vllm.config.model import ModelConfig - from vllm.model_executor.layers.quantization import QuantizationConfig + from vllm.config import VllmConfig def _is_squared(node: object, x: fx.Node) -> bool: @@ -71,10 +71,8 @@ def _is_one_plus(node: object) -> bool: def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool: """Does the forward compute anything after `node` before returning?""" - output = find_node(graph, lambda n: n.op == "output") - if output is None or not output.args: - return False - return peel(output.args[0]) is not node + value = output_value(graph) + return value is not None and peel(value) is not node class TPAwareNormMixin(nn.Module): @@ -185,17 +183,14 @@ class RMSNormFuser(BaseFuser): return eps return None - def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool: + def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: return True def fuse( - self, - module: nn.Module, - prefix: str, - model_config: "ModelConfig", - quant_config: "QuantizationConfig", + self, module: nn.Module, prefix: str, vllm_config: "VllmConfig" ) -> nn.Module: """Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp.""" + model_config = vllm_config.model_config weight = getattr(module, "weight", None) hidden_size = ( weight.size(0) if weight is not None else model_config.get_hidden_size() diff --git a/vllm/model_executor/models/transformers/fx_utils.py b/vllm/model_executor/models/transformers/fx_utils.py index 0e043941d8..2dbc3a499d 100644 --- a/vllm/model_executor/models/transformers/fx_utils.py +++ b/vllm/model_executor/models/transformers/fx_utils.py @@ -9,10 +9,13 @@ stays live Python. `fusion.py` builds the concrete fusion patterns on top. """ import ast +import contextlib import inspect import operator import textwrap from collections.abc import Callable +from itertools import chain +from unittest import mock import torch from torch import fx, nn @@ -22,46 +25,64 @@ from vllm.logger import init_logger logger = init_logger(__name__) +_UNKNOWN = object() +"""Sentinel meta value for proxies whose concrete value could not be inferred. +Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`).""" -def _infer_len(node: fx.Node) -> int | None: - """Concrete length of a proxy's value, inferred from its node chain. +_MODULE_CALL = nn.Module.__call__ +"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record +`call_module` nodes; meta execution must call modules for real.""" - Lets tracing pass through the shape unpacks and `*`-splats (e.g. - `(*input_shape, -1, head_dim)`) that precede the patterns in HF attention. - """ - # `x.shape` has the rank of `x`, when known - if ( - node.op == "call_function" - and node.target is getattr - and node.args[1] == "shape" - and (rank := _rank(node.args[0])) is not None - ): - return rank - # Slices of known-length values - if node.op == "call_function" and node.target is operator.getitem: - src_len = _infer_len(node.args[0]) - index = node.args[1] - if src_len is not None and isinstance(index, slice): - return len(range(*index.indices(src_len))) + +def is_leaf_call(node: object) -> bool: + """Is node a call recorded by `_as_leaf_call` (e.g. an attention interface).""" + return isinstance(node, fx.Node) and node.meta.get("leaf_call", False) + + +def _reference_weight(module: nn.Module) -> torch.Tensor | None: + """A weight whose trailing dim is the module's hidden size. + + Linears and 2-D gate weights are `[out, hidden]`; norm weights are + `[hidden]`. Used to fabricate a placeholder input of matching size/dtype.""" + for child in module.modules(): + if isinstance(child, nn.Linear): + return child.weight + for param in module.parameters(): + if param.ndim in (1, 2): + return param return None -def _rank(node: fx.Node) -> int | None: - """The tensor rank of `node`'s value, if known.""" - # vLLM always feeds the model [1, seq_len, hidden_size] hidden states - if node.op == "placeholder" and node.target == "hidden_states": - return 3 - return None +class _MetaProxy(fx.Proxy): + """Proxy carrying the meta-tensor value of the traced expression. + Shape questions (`len`, iteration, `.shape` unpacks) are answered by + executing each op on the meta values, so PyTorch's meta kernels are the + single source of shape inference — no per-op rules.""" -class _SizedProxy(fx.Proxy): - """Proxy whose `len` is inferred from the graph (see `_infer_len`).""" + meta: object = _UNKNOWN def __len__(self) -> int: - length = _infer_len(self.node) - if length is None: - return super().__len__() - return length + if self.meta is not _UNKNOWN: + return len(self.meta) + return super().__len__() # type: ignore[misc] + + def __getattr__(self, k: str) -> "_MetaAttribute": + return _MetaAttribute(self, k) + + +class _MetaAttribute(_MetaProxy, fx.proxy.Attribute): + """Attribute proxy (e.g. `x.shape`) carrying its meta value. + + `Proxy.__getattr__` constructs `Attribute` directly, bypassing + `Tracer.proxy`, so the meta value must be grafted on here too.""" + + def __init__(self, root: fx.Proxy, attr: str): + super().__init__(root, attr) + root_meta = getattr(root, "meta", _UNKNOWN) + if root_meta is not _UNKNOWN: + with contextlib.suppress(Exception): + self.meta = getattr(root_meta, attr) class _AllLeafTracer(fx.Tracer): @@ -69,31 +90,159 @@ class _AllLeafTracer(fx.Tracer): Each child stays one `call_module` node, so matching sees the module's own forward structure (activations aren't decomposed into e.g. `sigmoid * x`). - `iter` traces through the leading shape unpacks (see `_infer_len`); anything - else untraceable ends the trace early and the partial graph is matched. + Every traced op is also executed on meta tensors (see `_MetaProxy`) so + shape unpacks and `*`-splats trace through; anything else untraceable ends + the trace early and the partial graph is matched. """ + varkw: str | None = None + """Name of the traced forward's `**kwargs` parameter, if any.""" + def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool: return True def proxy(self, node: fx.Node) -> fx.Proxy: - return _SizedProxy(node, self) + return _MetaProxy(node, self) + + def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs): + proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs) + if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN: + # A failure stays _UNKNOWN; only fatal if a shape question is asked. + with contextlib.suppress(Exception): + proxy.meta = self._infer_meta(kind, target, args, kwargs) + return proxy + + def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict): + """Execute the op on meta tensors; PyTorch infers the output value.""" + if kind == "placeholder": + # vLLM always feeds the model [1, seq_len, hidden_size] hidden states. + weight = _reference_weight(self.root) + if str(target) == "hidden_states" and weight is not None: + return torch.empty( + 1, 8, weight.shape[-1], dtype=weight.dtype, device="meta" + ) + return _UNKNOWN + if kind == "get_attr": + value = operator.attrgetter(str(target))(self.root) + if isinstance(value, torch.Tensor): + value = torch.empty_like(value, device="meta") + return value + unknown = False + + def meta_of(arg: object) -> object: + nonlocal unknown + if isinstance(arg, fx.Proxy): + meta = getattr(arg, "meta", _UNKNOWN) + unknown = unknown or meta is _UNKNOWN + return meta + return arg + + meta_args = fx.node.map_aggregate(args, meta_of) + meta_kwargs = fx.node.map_aggregate(kwargs, meta_of) + if unknown: + return _UNKNOWN + if kind == "call_function": + return target(*meta_args, **meta_kwargs) + if kind == "call_method": + receiver, *rest = meta_args + return getattr(receiver, str(target))(*rest, **meta_kwargs) + if kind == "call_module": + # Run the child's forward with all its state on "meta", without + # mutating it (at match time params may be meta but buffers real). + # fx patches `nn.Module.__call__` while tracing; restore the real + # one so this execution is not itself recorded. + child = self.root.get_submodule(str(target)) + state = { + name: torch.empty_like(tensor, device="meta") + for name, tensor in chain( + child.named_parameters(), child.named_buffers() + ) + } + with mock.patch.object(nn.Module, "__call__", _MODULE_CALL): + return torch.func.functional_call(child, state, meta_args, meta_kwargs) + return _UNKNOWN + + def _is_varkw(self, node: object) -> bool: + return ( + isinstance(node, fx.Node) + and node.op == "placeholder" + and str(node.target).lstrip("*") == self.varkw + ) def iter(self, obj: fx.Proxy): - length = _infer_len(obj.node) - if length is None: + # Assume kwargs is always empty to simplify tracing. + node = obj.node + if self._is_varkw(node) or ( + node.op == "call_method" + and node.target == "keys" + and self._is_varkw(node.args[0]) + ): + return iter(()) + meta = getattr(obj, "meta", _UNKNOWN) + if meta is _UNKNOWN: return super().iter(obj) - return iter([obj[i] for i in range(length)]) + return iter([obj[i] for i in range(len(meta))]) + + +def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable: + """Wrap any callable so tracing records it as one opaque `call_function` node. + + Lets the trace continue past untraceable bodies. Only the proxy arguments carry into + the node's dataflow; the rest are dropped rather than lifted into the graph. + `length` declares how many values the callable returns, so unpacking its result also + traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough. + """ + + def leaf(*args, **kwargs): + proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy)) + if not proxies: + return fn(*args, **kwargs) + proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {}) + proxy.node.meta["leaf_call"] = True + if length is not None: + # The body never executes, so fabricate a value of the declared length. + proxy.meta = (_UNKNOWN,) * length + return proxy + + return leaf + + +def _leaf_attention_interfaces(): + """Patch `AttentionInterface.get_interface` so traced forwards see a leaf node. + + `vllm_attention_function` needs runtime context so it is untraceable. + Every interface returns `(attn_output, attn_weights)`.""" + from transformers.modeling_utils import AttentionInterface + + original = AttentionInterface.get_interface + + def get_interface(self, *args, **kwargs): + return _as_leaf_call(original(self, *args, **kwargs), length=2) + + return mock.patch.object(AttentionInterface, "get_interface", get_interface) def trace(module: nn.Module) -> fx.Graph | None: - """Trace `module.forward`, returning the partial graph on failure. - - The graph is only evidence for matching, and the patterns sit at the top of - their forwards, so a trace that fails partway can still be matched.""" + """Trace `module.forward`, returning the partial graph on failure.""" + parameters = forward_parameters(type(module)) + # vLLM never passes `past_key_values` so it is always the default value of `None`. + # Make this concrete to simplify tracing. + concrete_args = None + if "past_key_values" in parameters: + concrete_args = {"past_key_values": None} + # Get the name of the kwargs parameter passed to module.forward (usually "kwargs") tracer = _AllLeafTracer() + tracer.varkw = next( + ( + p.name + for p in parameters.values() + if p.kind is inspect.Parameter.VAR_KEYWORD + ), + None, + ) try: - return tracer.trace(module) + with _leaf_attention_interfaces(): + return tracer.trace(module, concrete_args=concrete_args) except Exception as exc: logger.debug("Could not fully trace %s: %s", type(module), exc) return getattr(tracer, "graph", None) @@ -129,20 +278,27 @@ def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]: return funcdef, fn +def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]: + """`cls.forward`'s signature parameters, or empty if uninspectable.""" + try: + return dict(inspect.signature(cls.forward).parameters) + except (TypeError, ValueError): + return {} + + def forward_input_count(cls: type[nn.Module]) -> int: """The number of tensor inputs `cls.forward` declares, excluding `self` and any `*args`/`**kwargs`. Read from the signature, so it is independent of whether the trace completes (unlike counting placeholders).""" - try: - params = list(inspect.signature(cls.forward).parameters.values())[1:] - except (ValueError, TypeError): + params = list(forward_parameters(cls).values()) + if not params: return 1 # uninspectable: assume a single input and let matching decide fixed = ( inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD, inspect.Parameter.KEYWORD_ONLY, ) - return sum(1 for p in params if p.kind in fixed) + return sum(1 for p in params[1:] if p.kind in fixed) def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable: @@ -227,6 +383,42 @@ def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node return next((n for n in graph.nodes if predicate(n)), None) +def output_value(graph: fx.Graph) -> object | None: + """The value the graph's `output` node returns, if the trace reached one.""" + output = find_node(graph, lambda n: n.op == "output") + if output is None or not output.args: + return None + return output.args[0] + + +def upstream_linear(node: object, module: nn.Module) -> fx.Node | None: + """Nearest linear producing `node`, walking back through splits/reshapes. + + Never walks through a leaf call (e.g. an attention interface): its inputs + are what attention consumes, not what produced the value.""" + stack = [node] + seen: set[fx.Node] = set() + while stack: + current = stack.pop() + if not isinstance(current, fx.Node) or current in seen: + continue + seen.add(current) + if is_linear(current, module): + return current + if current.op in ("call_function", "call_method") and not is_leaf_call(current): + stack.extend(current.args) + return None + + +def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None: + """Name of the Linear producing the graph's (first) output value.""" + value = output_value(graph) + if isinstance(value, (tuple, list)) and value: + value = value[0] + linear = upstream_linear(value, module) + return None if linear is None else str(linear.target) + + def is_linear(node: fx.Node, module: nn.Module) -> bool: """Is node `nn.Linear.__call__()`.""" return node.op == "call_module" and isinstance(