Improve Transformers modelling backend fx tracer (#49957)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -368,12 +368,12 @@ def test_qkv_identifies_output_projection():
|
||||
assert get_fuser(PerHeadQKNormAttention()).o_name == "o_proj"
|
||||
|
||||
|
||||
def test_fuser_is_cached_per_class():
|
||||
def test_fuser_is_cached_per_class_and_structure():
|
||||
with torch.device("meta"):
|
||||
fuser_a = get_fuser(GLUMLP())
|
||||
fuser_b = get_fuser(GLUMLP())
|
||||
assert fuser_a is fuser_b
|
||||
assert GLUMLP in get_fuser.cache
|
||||
assert any(key[0] is GLUMLP for key in get_fuser.cache)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("cls", [NotAnMLP, UntraceableMLP])
|
||||
@@ -460,8 +460,7 @@ def test_unfusable_modules_are_not_fused(cls, default_vllm_config):
|
||||
fuser = get_fuser(module)
|
||||
# Either no pattern matches the class, or this instance fails validation
|
||||
# (`recursive_replace` gates fusion and its weight mappings on `validate`)
|
||||
model_config = default_vllm_config.model_config
|
||||
assert fuser is None or not fuser.validate(module, model_config)
|
||||
assert fuser is None or not fuser.validate(module, default_vllm_config)
|
||||
|
||||
|
||||
def test_act_and_mul_derived_from_module(default_vllm_config):
|
||||
|
||||
@@ -148,11 +148,13 @@ def test_rms_norm_builds_vllm_class(cls, expected, zero_centered, default_vllm_c
|
||||
|
||||
# `default_vllm_config` supplies the config context the CustomOp needs; the
|
||||
# weightless path reads hidden size from the model config, so stub it.
|
||||
model_config = SimpleNamespace(get_hidden_size=lambda: 16)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_hidden_size=lambda: 16)
|
||||
)
|
||||
with torch.device("meta"):
|
||||
module = cls()
|
||||
fuser = get_fuser(module)
|
||||
built = fuser.fuse(module, "norm", model_config, None)
|
||||
built = fuser.fuse(module, "norm", vllm_config)
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import (
|
||||
TPAwareNormMixin,
|
||||
)
|
||||
@@ -176,8 +178,9 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
fuser = get_fuser(module)
|
||||
assert isinstance(fuser, RMSNormFuser)
|
||||
assert not fuser.zero_centered
|
||||
model_config = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32)
|
||||
built = fuser.fuse(module, "norm", model_config, None)
|
||||
mc = SimpleNamespace(get_hidden_size=lambda: 16, dtype=torch.float32)
|
||||
vllm_config = SimpleNamespace(model_config=mc)
|
||||
built = fuser.fuse(module, "norm", vllm_config)
|
||||
assert isinstance(built, VLLMRMSNorm)
|
||||
assert built.variance_epsilon == torch.finfo(torch.float32).eps
|
||||
|
||||
@@ -185,11 +188,13 @@ def test_fused_rms_norm_op_default_eps(default_vllm_config):
|
||||
def test_eps_is_derived_per_instance(default_vllm_config):
|
||||
"""Two instances of the same norm class with different eps must fuse to their
|
||||
own eps: the type-cached fuser holds only structure, not this value."""
|
||||
model_config = SimpleNamespace(get_hidden_size=lambda: 16)
|
||||
vllm_config = SimpleNamespace(
|
||||
model_config=SimpleNamespace(get_hidden_size=lambda: 16)
|
||||
)
|
||||
with torch.device("meta"):
|
||||
for eps in (1e-5, 1e-6):
|
||||
module = RMSNorm(16, eps=eps)
|
||||
built = get_fuser(module).fuse(module, "norm", model_config, None)
|
||||
built = get_fuser(module).fuse(module, "norm", vllm_config)
|
||||
assert built.variance_epsilon == eps
|
||||
|
||||
|
||||
|
||||
@@ -104,6 +104,7 @@ class Base(
|
||||
super().__init__()
|
||||
logger.info("Using Transformers modeling backend.")
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.text_config = self.config.get_text_config()
|
||||
self.cache_config = vllm_config.cache_config
|
||||
@@ -357,7 +358,7 @@ class Base(
|
||||
tip = get_feature_request_tip(
|
||||
self.model_config.model, self.model_config.trust_remote_code
|
||||
)
|
||||
logger.warning(
|
||||
logger.warning_once(
|
||||
"%s does not define a pipeline parallel plan. The Transformers "
|
||||
"modeling backend will infer the split from the layers of %s in order "
|
||||
"of declaration and keep parameter-free modules on every rank. This "
|
||||
@@ -452,7 +453,7 @@ class Base(
|
||||
# Prefix the patterns because we always start from `self.model`
|
||||
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
|
||||
# Detect fusable patterns once per module class (cached, so this is cheap)
|
||||
fusers = Fusers(self.model, self.model_config)
|
||||
fusers = Fusers(self.model, self.vllm_config)
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
@@ -503,9 +504,7 @@ class Base(
|
||||
new_module = replace_conv_class(child_module)
|
||||
elif (fuser := fusers[child_module]) is not None:
|
||||
register_fusion(fuser, qual_name)
|
||||
new_module = fuser.fuse(
|
||||
child_module, qual_name, self.model_config, self.quant_config
|
||||
)
|
||||
new_module = fuser.fuse(child_module, qual_name, self.vllm_config)
|
||||
logger.info_once(fuser.info(child_name))
|
||||
_recursive_replace(new_module, prefix=qual_name)
|
||||
elif not isinstance(child_module, MoERunner):
|
||||
|
||||
@@ -25,14 +25,19 @@ from vllm.model_executor.models.transformers.fusers import (
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@cached(cache={}, key=type)
|
||||
def key(module: nn.Module) -> tuple:
|
||||
"""Cache key for `get_fuser`. Considers module type and its immediate children."""
|
||||
return (type(module), tuple(name for name, _ in module.named_children()))
|
||||
|
||||
|
||||
@cached(cache={}, key=key)
|
||||
def get_fuser(module: nn.Module) -> BaseFuser | None:
|
||||
"""The fuser for `type(module)` (cached per class), or `None` if no match."""
|
||||
"""The fuser for `module`'s class and shape (cached), or `None` if no match."""
|
||||
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
|
||||
# leaf module (raw tensor math, no submodules). Nothing else can match, and
|
||||
# tracing is skipped for it.
|
||||
@@ -48,11 +53,14 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
|
||||
try:
|
||||
fuser.update_forward(module)
|
||||
except Exception as exc:
|
||||
# An unrecognised source just means we cannot fuse here.
|
||||
logger.debug(
|
||||
"Could not rewrite %s for fusion: %s", type(module), exc
|
||||
"Attempted to fuse %s using %s but failed "
|
||||
"to update its forward method: %s",
|
||||
type(module),
|
||||
fuser_cls.__name__,
|
||||
exc,
|
||||
)
|
||||
return None
|
||||
continue
|
||||
return fuser
|
||||
# A norm we could not match structurally is left unfused; flag likely misses.
|
||||
if module.__class__.__name__.endswith("RMSNorm"):
|
||||
@@ -67,12 +75,12 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
|
||||
class Fusers(UserDict):
|
||||
"""Mapping from module class to fuser, for all fusable classes in a model."""
|
||||
|
||||
def __init__(self, model: nn.Module, model_config: "ModelConfig"):
|
||||
self.model_config = model_config
|
||||
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
|
||||
self.vllm_config = vllm_config
|
||||
super().__init__({type(m): get_fuser(m) for m in model.modules()})
|
||||
|
||||
def __getitem__(self, m: nn.Module) -> BaseFuser | None:
|
||||
fuser = self.data.get(type(m))
|
||||
if fuser is not None and fuser.validate(m, self.model_config):
|
||||
if fuser is not None and fuser.validate(m, self.vllm_config):
|
||||
return fuser
|
||||
return None
|
||||
|
||||
@@ -13,8 +13,7 @@ from torch import fx, nn
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -36,16 +35,12 @@ class BaseFuser(ABC):
|
||||
"""Match the pattern in `graph`, returning a fuser if found."""
|
||||
|
||||
@abstractmethod
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether this fuser can be applied to this `module` instance."""
|
||||
|
||||
@abstractmethod
|
||||
def fuse(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
@@ -123,24 +118,16 @@ class StackedFuser(BaseFuser):
|
||||
|
||||
@abstractmethod
|
||||
def update_attrs(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace `module`'s submodules with the merged module."""
|
||||
|
||||
def fuse(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, model_config, quant_config)
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
|
||||
@@ -33,8 +33,7 @@ from vllm.model_executor.models.transformers.utils import (
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -168,7 +167,7 @@ class GLUFuser(StackedFuser):
|
||||
replace_expr(funcdef, muls[0], act_call)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
act = module.get_submodule(self.act_name)
|
||||
if self._get_act_and_mul_name(act) is None:
|
||||
logger.debug("No AndMul equivalent for %s; skipping fusion", type(act))
|
||||
@@ -176,12 +175,9 @@ class GLUFuser(StackedFuser):
|
||||
return True
|
||||
|
||||
def update_attrs(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
|
||||
gate = module.get_submodule(self.gate_name)
|
||||
up = module.get_submodule(self.up_name)
|
||||
|
||||
@@ -17,6 +17,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
is_linear,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
@@ -26,8 +27,7 @@ from vllm.model_executor.models.transformers.utils import (
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -84,15 +84,16 @@ class QKVFuser(StackedFuser):
|
||||
return None
|
||||
q, k, v = qkv_nodes
|
||||
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
|
||||
attn_width = module.get_submodule(q.target).out_features
|
||||
candidates = [
|
||||
name
|
||||
for name, child in module.named_children()
|
||||
if isinstance(child, nn.Linear)
|
||||
and name not in names.values()
|
||||
and child.in_features == attn_width
|
||||
]
|
||||
names["o_name"] = candidates[0] if len(candidates) == 1 else None
|
||||
# o_proj produces the module's output.
|
||||
o_name = returned_linear(graph, module)
|
||||
# o_proj must be compatible with the q/k/v projections.
|
||||
if o_name in names.values() or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features
|
||||
!= module.get_submodule(q.target).out_features
|
||||
):
|
||||
o_name = None
|
||||
names["o_name"] = o_name
|
||||
return cls(source_cls=type(module).__name__, **names)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
@@ -149,12 +150,12 @@ class QKVFuser(StackedFuser):
|
||||
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
v = module.get_submodule(self.v_name)
|
||||
head_size = model_config.get_head_size()
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
compatible = (
|
||||
q.in_features == k.in_features == v.in_features
|
||||
and len({proj.bias is None for proj in (q, k, v)}) == 1
|
||||
@@ -167,13 +168,10 @@ class QKVFuser(StackedFuser):
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
head_size = model_config.get_head_size()
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
merged = QKVParallelLinear(
|
||||
|
||||
@@ -21,13 +21,13 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
forward_input_count,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config.model import ModelConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
def _is_squared(node: object, x: fx.Node) -> bool:
|
||||
@@ -71,10 +71,8 @@ def _is_one_plus(node: object) -> bool:
|
||||
|
||||
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
|
||||
"""Does the forward compute anything after `node` before returning?"""
|
||||
output = find_node(graph, lambda n: n.op == "output")
|
||||
if output is None or not output.args:
|
||||
return False
|
||||
return peel(output.args[0]) is not node
|
||||
value = output_value(graph)
|
||||
return value is not None and peel(value) is not node
|
||||
|
||||
|
||||
class TPAwareNormMixin(nn.Module):
|
||||
@@ -185,17 +183,14 @@ class RMSNormFuser(BaseFuser):
|
||||
return eps
|
||||
return None
|
||||
|
||||
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
return True
|
||||
|
||||
def fuse(
|
||||
self,
|
||||
module: nn.Module,
|
||||
prefix: str,
|
||||
model_config: "ModelConfig",
|
||||
quant_config: "QuantizationConfig",
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
|
||||
model_config = vllm_config.model_config
|
||||
weight = getattr(module, "weight", None)
|
||||
hidden_size = (
|
||||
weight.size(0) if weight is not None else model_config.get_hidden_size()
|
||||
|
||||
@@ -9,10 +9,13 @@ stays live Python. `fusion.py` builds the concrete fusion patterns on top.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import inspect
|
||||
import operator
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from itertools import chain
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
@@ -22,46 +25,64 @@ from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_UNKNOWN = object()
|
||||
"""Sentinel meta value for proxies whose concrete value could not be inferred.
|
||||
Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`)."""
|
||||
|
||||
def _infer_len(node: fx.Node) -> int | None:
|
||||
"""Concrete length of a proxy's value, inferred from its node chain.
|
||||
_MODULE_CALL = nn.Module.__call__
|
||||
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
|
||||
`call_module` nodes; meta execution must call modules for real."""
|
||||
|
||||
Lets tracing pass through the shape unpacks and `*`-splats (e.g.
|
||||
`(*input_shape, -1, head_dim)`) that precede the patterns in HF attention.
|
||||
"""
|
||||
# `x.shape` has the rank of `x`, when known
|
||||
if (
|
||||
node.op == "call_function"
|
||||
and node.target is getattr
|
||||
and node.args[1] == "shape"
|
||||
and (rank := _rank(node.args[0])) is not None
|
||||
):
|
||||
return rank
|
||||
# Slices of known-length values
|
||||
if node.op == "call_function" and node.target is operator.getitem:
|
||||
src_len = _infer_len(node.args[0])
|
||||
index = node.args[1]
|
||||
if src_len is not None and isinstance(index, slice):
|
||||
return len(range(*index.indices(src_len)))
|
||||
|
||||
def is_leaf_call(node: object) -> bool:
|
||||
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
|
||||
return isinstance(node, fx.Node) and node.meta.get("leaf_call", False)
|
||||
|
||||
|
||||
def _reference_weight(module: nn.Module) -> torch.Tensor | None:
|
||||
"""A weight whose trailing dim is the module's hidden size.
|
||||
|
||||
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
|
||||
for child in module.modules():
|
||||
if isinstance(child, nn.Linear):
|
||||
return child.weight
|
||||
for param in module.parameters():
|
||||
if param.ndim in (1, 2):
|
||||
return param
|
||||
return None
|
||||
|
||||
|
||||
def _rank(node: fx.Node) -> int | None:
|
||||
"""The tensor rank of `node`'s value, if known."""
|
||||
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states
|
||||
if node.op == "placeholder" and node.target == "hidden_states":
|
||||
return 3
|
||||
return None
|
||||
class _MetaProxy(fx.Proxy):
|
||||
"""Proxy carrying the meta-tensor value of the traced expression.
|
||||
|
||||
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
|
||||
executing each op on the meta values, so PyTorch's meta kernels are the
|
||||
single source of shape inference — no per-op rules."""
|
||||
|
||||
class _SizedProxy(fx.Proxy):
|
||||
"""Proxy whose `len` is inferred from the graph (see `_infer_len`)."""
|
||||
meta: object = _UNKNOWN
|
||||
|
||||
def __len__(self) -> int:
|
||||
length = _infer_len(self.node)
|
||||
if length is None:
|
||||
return super().__len__()
|
||||
return length
|
||||
if self.meta is not _UNKNOWN:
|
||||
return len(self.meta)
|
||||
return super().__len__() # type: ignore[misc]
|
||||
|
||||
def __getattr__(self, k: str) -> "_MetaAttribute":
|
||||
return _MetaAttribute(self, k)
|
||||
|
||||
|
||||
class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
|
||||
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
|
||||
|
||||
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too."""
|
||||
|
||||
def __init__(self, root: fx.Proxy, attr: str):
|
||||
super().__init__(root, attr)
|
||||
root_meta = getattr(root, "meta", _UNKNOWN)
|
||||
if root_meta is not _UNKNOWN:
|
||||
with contextlib.suppress(Exception):
|
||||
self.meta = getattr(root_meta, attr)
|
||||
|
||||
|
||||
class _AllLeafTracer(fx.Tracer):
|
||||
@@ -69,31 +90,159 @@ class _AllLeafTracer(fx.Tracer):
|
||||
|
||||
Each child stays one `call_module` node, so matching sees the module's own
|
||||
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
|
||||
`iter` traces through the leading shape unpacks (see `_infer_len`); anything
|
||||
else untraceable ends the trace early and the partial graph is matched.
|
||||
Every traced op is also executed on meta tensors (see `_MetaProxy`) so
|
||||
shape unpacks and `*`-splats trace through; anything else untraceable ends
|
||||
the trace early and the partial graph is matched.
|
||||
"""
|
||||
|
||||
varkw: str | None = None
|
||||
"""Name of the traced forward's `**kwargs` parameter, if any."""
|
||||
|
||||
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
|
||||
return True
|
||||
|
||||
def proxy(self, node: fx.Node) -> fx.Proxy:
|
||||
return _SizedProxy(node, self)
|
||||
return _MetaProxy(node, self)
|
||||
|
||||
def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs):
|
||||
proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs)
|
||||
if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN:
|
||||
# A failure stays _UNKNOWN; only fatal if a shape question is asked.
|
||||
with contextlib.suppress(Exception):
|
||||
proxy.meta = self._infer_meta(kind, target, args, kwargs)
|
||||
return proxy
|
||||
|
||||
def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict):
|
||||
"""Execute the op on meta tensors; PyTorch infers the output value."""
|
||||
if kind == "placeholder":
|
||||
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states.
|
||||
weight = _reference_weight(self.root)
|
||||
if str(target) == "hidden_states" and weight is not None:
|
||||
return torch.empty(
|
||||
1, 8, weight.shape[-1], dtype=weight.dtype, device="meta"
|
||||
)
|
||||
return _UNKNOWN
|
||||
if kind == "get_attr":
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = torch.empty_like(value, device="meta")
|
||||
return value
|
||||
unknown = False
|
||||
|
||||
def meta_of(arg: object) -> object:
|
||||
nonlocal unknown
|
||||
if isinstance(arg, fx.Proxy):
|
||||
meta = getattr(arg, "meta", _UNKNOWN)
|
||||
unknown = unknown or meta is _UNKNOWN
|
||||
return meta
|
||||
return arg
|
||||
|
||||
meta_args = fx.node.map_aggregate(args, meta_of)
|
||||
meta_kwargs = fx.node.map_aggregate(kwargs, meta_of)
|
||||
if unknown:
|
||||
return _UNKNOWN
|
||||
if kind == "call_function":
|
||||
return target(*meta_args, **meta_kwargs)
|
||||
if kind == "call_method":
|
||||
receiver, *rest = meta_args
|
||||
return getattr(receiver, str(target))(*rest, **meta_kwargs)
|
||||
if kind == "call_module":
|
||||
# Run the child's forward with all its state on "meta", without
|
||||
# mutating it (at match time params may be meta but buffers real).
|
||||
# fx patches `nn.Module.__call__` while tracing; restore the real
|
||||
# one so this execution is not itself recorded.
|
||||
child = self.root.get_submodule(str(target))
|
||||
state = {
|
||||
name: torch.empty_like(tensor, device="meta")
|
||||
for name, tensor in chain(
|
||||
child.named_parameters(), child.named_buffers()
|
||||
)
|
||||
}
|
||||
with mock.patch.object(nn.Module, "__call__", _MODULE_CALL):
|
||||
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
|
||||
return _UNKNOWN
|
||||
|
||||
def _is_varkw(self, node: object) -> bool:
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "placeholder"
|
||||
and str(node.target).lstrip("*") == self.varkw
|
||||
)
|
||||
|
||||
def iter(self, obj: fx.Proxy):
|
||||
length = _infer_len(obj.node)
|
||||
if length is None:
|
||||
# Assume kwargs is always empty to simplify tracing.
|
||||
node = obj.node
|
||||
if self._is_varkw(node) or (
|
||||
node.op == "call_method"
|
||||
and node.target == "keys"
|
||||
and self._is_varkw(node.args[0])
|
||||
):
|
||||
return iter(())
|
||||
meta = getattr(obj, "meta", _UNKNOWN)
|
||||
if meta is _UNKNOWN:
|
||||
return super().iter(obj)
|
||||
return iter([obj[i] for i in range(length)])
|
||||
return iter([obj[i] for i in range(len(meta))])
|
||||
|
||||
|
||||
def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable:
|
||||
"""Wrap any callable so tracing records it as one opaque `call_function` node.
|
||||
|
||||
Lets the trace continue past untraceable bodies. Only the proxy arguments carry into
|
||||
the node's dataflow; the rest are dropped rather than lifted into the graph.
|
||||
`length` declares how many values the callable returns, so unpacking its result also
|
||||
traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough.
|
||||
"""
|
||||
|
||||
def leaf(*args, **kwargs):
|
||||
proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy))
|
||||
if not proxies:
|
||||
return fn(*args, **kwargs)
|
||||
proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {})
|
||||
proxy.node.meta["leaf_call"] = True
|
||||
if length is not None:
|
||||
# The body never executes, so fabricate a value of the declared length.
|
||||
proxy.meta = (_UNKNOWN,) * length
|
||||
return proxy
|
||||
|
||||
return leaf
|
||||
|
||||
|
||||
def _leaf_attention_interfaces():
|
||||
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
|
||||
|
||||
`vllm_attention_function` needs runtime context so it is untraceable.
|
||||
Every interface returns `(attn_output, attn_weights)`."""
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
original = AttentionInterface.get_interface
|
||||
|
||||
def get_interface(self, *args, **kwargs):
|
||||
return _as_leaf_call(original(self, *args, **kwargs), length=2)
|
||||
|
||||
return mock.patch.object(AttentionInterface, "get_interface", get_interface)
|
||||
|
||||
|
||||
def trace(module: nn.Module) -> fx.Graph | None:
|
||||
"""Trace `module.forward`, returning the partial graph on failure.
|
||||
|
||||
The graph is only evidence for matching, and the patterns sit at the top of
|
||||
their forwards, so a trace that fails partway can still be matched."""
|
||||
"""Trace `module.forward`, returning the partial graph on failure."""
|
||||
parameters = forward_parameters(type(module))
|
||||
# vLLM never passes `past_key_values` so it is always the default value of `None`.
|
||||
# Make this concrete to simplify tracing.
|
||||
concrete_args = None
|
||||
if "past_key_values" in parameters:
|
||||
concrete_args = {"past_key_values": None}
|
||||
# Get the name of the kwargs parameter passed to module.forward (usually "kwargs")
|
||||
tracer = _AllLeafTracer()
|
||||
tracer.varkw = next(
|
||||
(
|
||||
p.name
|
||||
for p in parameters.values()
|
||||
if p.kind is inspect.Parameter.VAR_KEYWORD
|
||||
),
|
||||
None,
|
||||
)
|
||||
try:
|
||||
return tracer.trace(module)
|
||||
with _leaf_attention_interfaces():
|
||||
return tracer.trace(module, concrete_args=concrete_args)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not fully trace %s: %s", type(module), exc)
|
||||
return getattr(tracer, "graph", None)
|
||||
@@ -129,20 +278,27 @@ def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
|
||||
return funcdef, fn
|
||||
|
||||
|
||||
def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
|
||||
"""`cls.forward`'s signature parameters, or empty if uninspectable."""
|
||||
try:
|
||||
return dict(inspect.signature(cls.forward).parameters)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def forward_input_count(cls: type[nn.Module]) -> int:
|
||||
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
|
||||
any `*args`/`**kwargs`. Read from the signature, so it is independent of
|
||||
whether the trace completes (unlike counting placeholders)."""
|
||||
try:
|
||||
params = list(inspect.signature(cls.forward).parameters.values())[1:]
|
||||
except (ValueError, TypeError):
|
||||
params = list(forward_parameters(cls).values())
|
||||
if not params:
|
||||
return 1 # uninspectable: assume a single input and let matching decide
|
||||
fixed = (
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
return sum(1 for p in params if p.kind in fixed)
|
||||
return sum(1 for p in params[1:] if p.kind in fixed)
|
||||
|
||||
|
||||
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
|
||||
@@ -227,6 +383,42 @@ def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node
|
||||
return next((n for n in graph.nodes if predicate(n)), None)
|
||||
|
||||
|
||||
def output_value(graph: fx.Graph) -> object | None:
|
||||
"""The value the graph's `output` node returns, if the trace reached one."""
|
||||
output = find_node(graph, lambda n: n.op == "output")
|
||||
if output is None or not output.args:
|
||||
return None
|
||||
return output.args[0]
|
||||
|
||||
|
||||
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear producing `node`, walking back through splits/reshapes.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): its inputs
|
||||
are what attention consumes, not what produced the value."""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if not isinstance(current, fx.Node) or current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
if is_linear(current, module):
|
||||
return current
|
||||
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
|
||||
stack.extend(current.args)
|
||||
return None
|
||||
|
||||
|
||||
def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None:
|
||||
"""Name of the Linear producing the graph's (first) output value."""
|
||||
value = output_value(graph)
|
||||
if isinstance(value, (tuple, list)) and value:
|
||||
value = value[0]
|
||||
linear = upstream_linear(value, module)
|
||||
return None if linear is None else str(linear.target)
|
||||
|
||||
|
||||
def is_linear(node: fx.Node, module: nn.Module) -> bool:
|
||||
"""Is node `nn.Linear.__call__()`."""
|
||||
return node.op == "call_module" and isinstance(
|
||||
|
||||
Reference in New Issue
Block a user