Improve Transformers modelling backend fx tracer (#49957)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-27 12:51:16 +01:00
committed by GitHub
parent 77cba0259f
commit 92e8518d37
9 changed files with 307 additions and 128 deletions
@@ -13,8 +13,7 @@ from torch import fx, nn
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
@dataclass
@@ -36,16 +35,12 @@ class BaseFuser(ABC):
"""Match the pattern in `graph`, returning a fuser if found."""
@abstractmethod
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether this fuser can be applied to this `module` instance."""
@abstractmethod
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built)."""
@@ -123,24 +118,16 @@ class StackedFuser(BaseFuser):
@abstractmethod
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace `module`'s submodules with the merged module."""
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, model_config, quant_config)
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@@ -33,8 +33,7 @@ from vllm.model_executor.models.transformers.utils import (
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
logger = init_logger(__name__)
@@ -168,7 +167,7 @@ class GLUFuser(StackedFuser):
replace_expr(funcdef, muls[0], act_call)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
act = module.get_submodule(self.act_name)
if self._get_act_and_mul_name(act) is None:
logger.debug("No AndMul equivalent for %s; skipping fusion", type(act))
@@ -176,12 +175,9 @@ class GLUFuser(StackedFuser):
return True
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
gate = module.get_submodule(self.gate_name)
up = module.get_submodule(self.up_name)
@@ -17,6 +17,7 @@ from vllm.model_executor.models.transformers.fx_utils import (
is_linear,
recover_forward,
replace_expr,
returned_linear,
single_self_call,
)
from vllm.model_executor.models.transformers.utils import (
@@ -26,8 +27,7 @@ from vllm.model_executor.models.transformers.utils import (
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
logger = init_logger(__name__)
@@ -84,15 +84,16 @@ class QKVFuser(StackedFuser):
return None
q, k, v = qkv_nodes
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
attn_width = module.get_submodule(q.target).out_features
candidates = [
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
and name not in names.values()
and child.in_features == attn_width
]
names["o_name"] = candidates[0] if len(candidates) == 1 else None
# o_proj produces the module's output.
o_name = returned_linear(graph, module)
# o_proj must be compatible with the q/k/v projections.
if o_name in names.values() or (
o_name is not None
and module.get_submodule(o_name).in_features
!= module.get_submodule(q.target).out_features
):
o_name = None
names["o_name"] = o_name
return cls(source_cls=type(module).__name__, **names)
def update_forward(self, module: nn.Module) -> None:
@@ -149,12 +150,12 @@ class QKVFuser(StackedFuser):
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
v = module.get_submodule(self.v_name)
head_size = model_config.get_head_size()
head_size = vllm_config.model_config.get_head_size()
compatible = (
q.in_features == k.in_features == v.in_features
and len({proj.bias is None for proj in (q, k, v)}) == 1
@@ -167,13 +168,10 @@ class QKVFuser(StackedFuser):
return compatible
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
head_size = model_config.get_head_size()
quant_config = vllm_config.quant_config
head_size = vllm_config.model_config.get_head_size()
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
merged = QKVParallelLinear(
@@ -21,13 +21,13 @@ from vllm.model_executor.models.transformers.fx_utils import (
find_node,
forward_input_count,
is_op,
output_value,
peel,
trace,
)
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
from vllm.config import VllmConfig
def _is_squared(node: object, x: fx.Node) -> bool:
@@ -71,10 +71,8 @@ def _is_one_plus(node: object) -> bool:
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
"""Does the forward compute anything after `node` before returning?"""
output = find_node(graph, lambda n: n.op == "output")
if output is None or not output.args:
return False
return peel(output.args[0]) is not node
value = output_value(graph)
return value is not None and peel(value) is not node
class TPAwareNormMixin(nn.Module):
@@ -185,17 +183,14 @@ class RMSNormFuser(BaseFuser):
return eps
return None
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
return True
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
model_config = vllm_config.model_config
weight = getattr(module, "weight", None)
hidden_size = (
weight.size(0) if weight is not None else model_config.get_hidden_size()