Improve Transformers modelling backend fx tracer (#49957)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-27 12:51:16 +01:00
committed by GitHub
parent 77cba0259f
commit 92e8518d37
9 changed files with 307 additions and 128 deletions
@@ -368,12 +368,12 @@ def test_qkv_identifies_output_projection():
assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
def test_fuser_is_cached_per_class():
def test_fuser_is_cached_per_class_and_structure():
with torch.device("meta"):
fuser_a = get_fuser(GLUMLP())
fuser_b = get_fuser(GLUMLP())
assert fuser_a is fuser_b
assert GLUMLP in get_fuser.cache
assert any(key[0] is GLUMLP for key in get_fuser.cache)
@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP])
@@ -460,8 +460,7 @@ def test_unfusable_modules_are_not_fused(cls, default_vllm_config):
fuser = get_fuser(module)
# Either no pattern matches the class, or this instance fails validation
# (`recursive_replace` gates fusion and its weight mappings on `validate`)
model_config = default_vllm_config.model_config
assert fuser is None or not fuser.validate(module, model_config)
assert fuser is None or not fuser.validate(module, default_vllm_config)
def test_act_and_mul_derived_from_module(default_vllm_config):
@@ -148,11 +148,13 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c
# `default_vllm_config` supplies the config context the CustomOp needs; the
# weightless path reads hidden size from the model config, so stub it.
model_config = SimpleNamespace(get_hidden_size=lambda: 16)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: 16)
)
with torch.device("meta"):
module = cls()
fuser = get_fuser(module)
built = fuser.fuse(module, "norm", model_config, None)
built = fuser.fuse(module, "norm", vllm_config)
from vllm.model_executor.models.transformers.fusers.rms_norm import (
TPAwareNormMixin,
)
@@ -176,8 +178,9 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
fuser = get_fuser(module)
assert isinstance(fuser, RMSNormFuser)
assert not fuser.zero_centered
model_config = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32)
built = fuser.fuse(module, "norm", model_config, None)
mc = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32)
vllm_config = SimpleNamespace(model_config=mc)
built = fuser.fuse(module, "norm", vllm_config)
assert isinstance(built, VLLMRMSNorm)
assert built.variance_epsilon == torch.finfo(torch.float32).eps
@@ -185,11 +188,13 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
def test_eps_is_derived_per_instance(default_vllm_config):
"""Two instances of the same norm class with different eps must fuse to their
own eps: the type-cached fuser holds only structure, not this value."""
model_config = SimpleNamespace(get_hidden_size=lambda: 16)
vllm_config = SimpleNamespace(
model_config=SimpleNamespace(get_hidden_size=lambda: 16)
)
with torch.device("meta"):
for eps in (1e-5, 1e-6):
module = RMSNorm(16, eps=eps)
built = get_fuser(module).fuse(module, "norm", model_config, None)
built = get_fuser(module).fuse(module, "norm", vllm_config)
assert built.variance_epsilon == eps
@@ -104,6 +104,7 @@ class Base(
super().__init__()
logger.info("Using Transformers modeling backend.")
self.vllm_config = vllm_config
self.config = vllm_config.model_config.hf_config
self.text_config = self.config.get_text_config()
self.cache_config = vllm_config.cache_config
@@ -357,7 +358,7 @@ class Base(
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
logger.warning(
logger.warning_once(
"%s does not define a pipeline parallel plan. The Transformers "
"modeling backend will infer the split from the layers of %s in order "
"of declaration and keep parameter-free modules on every rank. This "
@@ -452,7 +453,7 @@ class Base(
# Prefix the patterns because we always start from `self.model`
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
# Detect fusable patterns once per module class (cached, so this is cheap)
fusers = Fusers(self.model, self.model_config)
fusers = Fusers(self.model, self.vllm_config)
def register_fusion(fuser: BaseFuser, prefix: str):
"""Register a fused layer's mappings just before it is built."""
@@ -503,9 +504,7 @@ class Base(
new_module = replace_conv_class(child_module)
elif (fuser := fusers[child_module]) is not None:
register_fusion(fuser, qual_name)
new_module = fuser.fuse(
child_module, qual_name, self.model_config, self.quant_config
)
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
@@ -25,14 +25,19 @@ from vllm.model_executor.models.transformers.fusers import (
from vllm.model_executor.models.transformers.fx_utils import trace
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.config import VllmConfig
logger = init_logger(__name__)
@cached(cache={}, key=type)
def key(module: nn.Module) -> tuple:
"""Cache key for `get_fuser`. Considers module type and its immediate children."""
return (type(module), tuple(name for name, _ in module.named_children()))
@cached(cache={}, key=key)
def get_fuser(module: nn.Module) -> BaseFuser | None:
"""The fuser for `type(module)` (cached per class), or `None` if no match."""
"""The fuser for `module`'s class and shape (cached), or `None` if no match."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
@@ -48,11 +53,14 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
try:
fuser.update_forward(module)
except Exception as exc:
# An unrecognised source just means we cannot fuse here.
logger.debug(
"Could not rewrite %s for fusion: %s", type(module), exc
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
return None
continue
return fuser
# A norm we could not match structurally is left unfused; flag likely misses.
if module.__class__.__name__.endswith("RMSNorm"):
@@ -67,12 +75,12 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
class Fusers(UserDict):
"""Mapping from module class to fuser, for all fusable classes in a model."""
def __init__(self, model: nn.Module, model_config: "ModelConfig"):
self.model_config = model_config
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
self.vllm_config = vllm_config
super().__init__({type(m): get_fuser(m) for m in model.modules()})
def __getitem__(self, m: nn.Module) -> BaseFuser | None:
fuser = self.data.get(type(m))
if fuser is not None and fuser.validate(m, self.model_config):
if fuser is not None and fuser.validate(m, self.vllm_config):
return fuser
return None
@@ -13,8 +13,7 @@ from torch import fx, nn
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
@dataclass
@@ -36,16 +35,12 @@ class BaseFuser(ABC):
"""Match the pattern in `graph`, returning a fuser if found."""
@abstractmethod
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether this fuser can be applied to this `module` instance."""
@abstractmethod
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built)."""
@@ -123,24 +118,16 @@ class StackedFuser(BaseFuser):
@abstractmethod
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace `module`'s submodules with the merged module."""
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, model_config, quant_config)
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@@ -33,8 +33,7 @@ from vllm.model_executor.models.transformers.utils import (
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
logger = init_logger(__name__)
@@ -168,7 +167,7 @@ class GLUFuser(StackedFuser):
replace_expr(funcdef, muls[0], act_call)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
act = module.get_submodule(self.act_name)
if self._get_act_and_mul_name(act) is None:
logger.debug("No AndMul equivalent for %s; skipping fusion", type(act))
@@ -176,12 +175,9 @@ class GLUFuser(StackedFuser):
return True
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
gate = module.get_submodule(self.gate_name)
up = module.get_submodule(self.up_name)
@@ -17,6 +17,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
is_linear,
recover_forward,
replace_expr,
returned_linear,
single_self_call,
)
from vllm.model_executor.models.transformers.utils import (
@@ -26,8 +27,7 @@ from vllm.model_executor.models.transformers.utils import (
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
logger = init_logger(__name__)
@@ -84,15 +84,16 @@ class QKVFuser(StackedFuser):
return None
q, k, v = qkv_nodes
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
attn_width = module.get_submodule(q.target).out_features
candidates = [
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
and name not in names.values()
and child.in_features == attn_width
]
names["o_name"] = candidates[0] if len(candidates) == 1 else None
# o_proj produces the module's output.
o_name = returned_linear(graph, module)
# o_proj must be compatible with the q/k/v projections.
if o_name in names.values() or (
o_name is not None
and module.get_submodule(o_name).in_features
!= module.get_submodule(q.target).out_features
):
o_name = None
names["o_name"] = o_name
return cls(source_cls=type(module).__name__, **names)
def update_forward(self, module: nn.Module) -> None:
@@ -149,12 +150,12 @@ class QKVFuser(StackedFuser):
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
v = module.get_submodule(self.v_name)
head_size = model_config.get_head_size()
head_size = vllm_config.model_config.get_head_size()
compatible = (
q.in_features == k.in_features == v.in_features
and len({proj.bias is None for proj in (q, k, v)}) == 1
@@ -167,13 +168,10 @@ class QKVFuser(StackedFuser):
return compatible
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
head_size = model_config.get_head_size()
quant_config = vllm_config.quant_config
head_size = vllm_config.model_config.get_head_size()
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
merged = QKVParallelLinear(
@@ -21,13 +21,13 @@ from vllm.model_executor.models.transformers.fx_utils import (
find_node,
forward_input_count,
is_op,
output_value,
peel,
trace,
)
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
def _is_squared(node: object, x: fx.Node) -> bool:
@@ -71,10 +71,8 @@ def _is_one_plus(node: object) -> bool:
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
"""Does the forward compute anything after `node` before returning?"""
output = find_node(graph, lambda n: n.op == "output")
if output is None or not output.args:
return False
return peel(output.args[0]) is not node
value = output_value(graph)
return value is not None and peel(value) is not node
class TPAwareNormMixin(nn.Module):
@@ -185,17 +183,14 @@ class RMSNormFuser(BaseFuser):
return eps
return None
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
return True
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
model_config = vllm_config.model_config
weight = getattr(module, "weight", None)
hidden_size = (
weight.size(0) if weight is not None else model_config.get_hidden_size()
@@ -9,10 +9,13 @@ stays live Python. `fusion.py` builds the concrete fusion patterns on top.
"""
import ast
import contextlib
import inspect
import operator
import textwrap
from collections.abc import Callable
from itertools import chain
from unittest import mock
import torch
from torch import fx, nn
@@ -22,46 +25,64 @@ from vllm.logger import init_logger
logger = init_logger(__name__)
_UNKNOWN = object()
"""Sentinel meta value for proxies whose concrete value could not be inferred.
Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`)."""
def _infer_len(node: fx.Node) -> int | None:
"""Concrete length of a proxy's value, inferred from its node chain.
_MODULE_CALL = nn.Module.__call__
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
`call_module` nodes; meta execution must call modules for real."""
Lets tracing pass through the shape unpacks and `*`-splats (e.g.
`(*input_shape, -1, head_dim)`) that precede the patterns in HF attention.
"""
# `x.shape` has the rank of `x`, when known
if (
node.op == "call_function"
and node.target is getattr
and node.args[1] == "shape"
and (rank := _rank(node.args[0])) is not None
):
return rank
# Slices of known-length values
if node.op == "call_function" and node.target is operator.getitem:
src_len = _infer_len(node.args[0])
index = node.args[1]
if src_len is not None and isinstance(index, slice):
return len(range(*index.indices(src_len)))
def is_leaf_call(node: object) -> bool:
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
return isinstance(node, fx.Node) and node.meta.get("leaf_call", False)
def _reference_weight(module: nn.Module) -> torch.Tensor | None:
"""A weight whose trailing dim is the module's hidden size.
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
for child in module.modules():
if isinstance(child, nn.Linear):
return child.weight
for param in module.parameters():
if param.ndim in (1, 2):
return param
return None
def _rank(node: fx.Node) -> int | None:
"""The tensor rank of `node`'s value, if known."""
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states
if node.op == "placeholder" and node.target == "hidden_states":
return 3
return None
class _MetaProxy(fx.Proxy):
"""Proxy carrying the meta-tensor value of the traced expression.
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
executing each op on the meta values, so PyTorch's meta kernels are the
single source of shape inference — no per-op rules."""
class _SizedProxy(fx.Proxy):
"""Proxy whose `len` is inferred from the graph (see `_infer_len`)."""
meta: object = _UNKNOWN
def __len__(self) -> int:
length = _infer_len(self.node)
if length is None:
return super().__len__()
return length
if self.meta is not _UNKNOWN:
return len(self.meta)
return super().__len__() # type: ignore[misc]
def __getattr__(self, k: str) -> "_MetaAttribute":
return _MetaAttribute(self, k)
class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
`Tracer.proxy`, so the meta value must be grafted on here too."""
def __init__(self, root: fx.Proxy, attr: str):
super().__init__(root, attr)
root_meta = getattr(root, "meta", _UNKNOWN)
if root_meta is not _UNKNOWN:
with contextlib.suppress(Exception):
self.meta = getattr(root_meta, attr)
class _AllLeafTracer(fx.Tracer):
@@ -69,31 +90,159 @@ class _AllLeafTracer(fx.Tracer):
Each child stays one `call_module` node, so matching sees the module's own
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
`iter` traces through the leading shape unpacks (see `_infer_len`); anything
else untraceable ends the trace early and the partial graph is matched.
Every traced op is also executed on meta tensors (see `_MetaProxy`) so
shape unpacks and `*`-splats trace through; anything else untraceable ends
the trace early and the partial graph is matched.
"""
varkw: str | None = None
"""Name of the traced forward's `**kwargs` parameter, if any."""
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
return True
def proxy(self, node: fx.Node) -> fx.Proxy:
return _SizedProxy(node, self)
return _MetaProxy(node, self)
def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs):
proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs)
if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN:
# A failure stays _UNKNOWN; only fatal if a shape question is asked.
with contextlib.suppress(Exception):
proxy.meta = self._infer_meta(kind, target, args, kwargs)
return proxy
def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict):
"""Execute the op on meta tensors; PyTorch infers the output value."""
if kind == "placeholder":
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states.
weight = _reference_weight(self.root)
if str(target) == "hidden_states" and weight is not None:
return torch.empty(
1, 8, weight.shape[-1], dtype=weight.dtype, device="meta"
)
return _UNKNOWN
if kind == "get_attr":
value = operator.attrgetter(str(target))(self.root)
if isinstance(value, torch.Tensor):
value = torch.empty_like(value, device="meta")
return value
unknown = False
def meta_of(arg: object) -> object:
nonlocal unknown
if isinstance(arg, fx.Proxy):
meta = getattr(arg, "meta", _UNKNOWN)
unknown = unknown or meta is _UNKNOWN
return meta
return arg
meta_args = fx.node.map_aggregate(args, meta_of)
meta_kwargs = fx.node.map_aggregate(kwargs, meta_of)
if unknown:
return _UNKNOWN
if kind == "call_function":
return target(*meta_args, **meta_kwargs)
if kind == "call_method":
receiver, *rest = meta_args
return getattr(receiver, str(target))(*rest, **meta_kwargs)
if kind == "call_module":
# Run the child's forward with all its state on "meta", without
# mutating it (at match time params may be meta but buffers real).
# fx patches `nn.Module.__call__` while tracing; restore the real
# one so this execution is not itself recorded.
child = self.root.get_submodule(str(target))
state = {
name: torch.empty_like(tensor, device="meta")
for name, tensor in chain(
child.named_parameters(), child.named_buffers()
)
}
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
return _UNKNOWN
def _is_varkw(self, node: object) -> bool:
return (
isinstance(node, fx.Node)
and node.op == "placeholder"
and str(node.target).lstrip("*") == self.varkw
)
def iter(self, obj: fx.Proxy):
length = _infer_len(obj.node)
if length is None:
# Assume kwargs is always empty to simplify tracing.
node = obj.node
if self._is_varkw(node) or (
node.op == "call_method"
and node.target == "keys"
and self._is_varkw(node.args[0])
):
return iter(())
meta = getattr(obj, "meta", _UNKNOWN)
if meta is _UNKNOWN:
return super().iter(obj)
return iter([obj[i] for i in range(length)])
return iter([obj[i] for i in range(len(meta))])
def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable:
"""Wrap any callable so tracing records it as one opaque `call_function` node.
Lets the trace continue past untraceable bodies. Only the proxy arguments carry into
the node's dataflow; the rest are dropped rather than lifted into the graph.
`length` declares how many values the callable returns, so unpacking its result also
traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough.
"""
def leaf(*args, **kwargs):
proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy))
if not proxies:
return fn(*args, **kwargs)
proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {})
proxy.node.meta["leaf_call"] = True
if length is not None:
# The body never executes, so fabricate a value of the declared length.
proxy.meta = (_UNKNOWN,) * length
return proxy
return leaf
def _leaf_attention_interfaces():
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
`vllm_attention_function` needs runtime context so it is untraceable.
Every interface returns `(attn_output, attn_weights)`."""
from transformers.modeling_utils import AttentionInterface
original = AttentionInterface.get_interface
def get_interface(self, *args, **kwargs):
return _as_leaf_call(original(self, *args, **kwargs), length=2)
return mock.patch.object(AttentionInterface, "get_interface", get_interface)
def trace(module: nn.Module) -> fx.Graph | None:
"""Trace `module.forward`, returning the partial graph on failure.
The graph is only evidence for matching, and the patterns sit at the top of
their forwards, so a trace that fails partway can still be matched."""
"""Trace `module.forward`, returning the partial graph on failure."""
parameters = forward_parameters(type(module))
# vLLM never passes `past_key_values` so it is always the default value of `None`.
# Make this concrete to simplify tracing.
concrete_args = None
if "past_key_values" in parameters:
concrete_args = {"past_key_values": None}
# Get the name of the kwargs parameter passed to module.forward (usually "kwargs")
tracer = _AllLeafTracer()
tracer.varkw = next(
(
p.name
for p in parameters.values()
if p.kind is inspect.Parameter.VAR_KEYWORD
),
None,
)
try:
return tracer.trace(module)
with _leaf_attention_interfaces():
return tracer.trace(module, concrete_args=concrete_args)
except Exception as exc:
logger.debug("Could not fully trace %s: %s", type(module), exc)
return getattr(tracer, "graph", None)
@@ -129,20 +278,27 @@ def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
return funcdef, fn
def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
"""`cls.forward`'s signature parameters, or empty if uninspectable."""
try:
return dict(inspect.signature(cls.forward).parameters)
except (TypeError, ValueError):
return {}
def forward_input_count(cls: type[nn.Module]) -> int:
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
any `*args`/`**kwargs`. Read from the signature, so it is independent of
whether the trace completes (unlike counting placeholders)."""
try:
params = list(inspect.signature(cls.forward).parameters.values())[1:]
except (ValueError, TypeError):
params = list(forward_parameters(cls).values())
if not params:
return 1 # uninspectable: assume a single input and let matching decide
fixed = (
inspect.Parameter.POSITIONAL_ONLY,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
)
return sum(1 for p in params if p.kind in fixed)
return sum(1 for p in params[1:] if p.kind in fixed)
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
@@ -227,6 +383,42 @@ def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node
return next((n for n in graph.nodes if predicate(n)), None)
def output_value(graph: fx.Graph) -> object | None:
"""The value the graph's `output` node returns, if the trace reached one."""
output = find_node(graph, lambda n: n.op == "output")
if output is None or not output.args:
return None
return output.args[0]
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
"""Nearest linear producing `node`, walking back through splits/reshapes.
Never walks through a leaf call (e.g. an attention interface): its inputs
are what attention consumes, not what produced the value."""
stack = [node]
seen: set[fx.Node] = set()
while stack:
current = stack.pop()
if not isinstance(current, fx.Node) or current in seen:
continue
seen.add(current)
if is_linear(current, module):
return current
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
stack.extend(current.args)
return None
def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None:
"""Name of the Linear producing the graph's (first) output value."""
value = output_value(graph)
if isinstance(value, (tuple, list)) and value:
value = value[0]
linear = upstream_linear(value, module)
return None if linear is None else str(linear.target)
def is_linear(node: fx.Node, module: nn.Module) -> bool:
"""Is node `nn.Linear.__call__()`."""
return node.op == "call_module" and isinstance(