[Transformers backend] Enable QKV-Fuser for Gemma4 (#55690)

Signed-off-by: Thomas Ortner <boh@zurich.ibm.com>
Signed-off-by: Thomas Ortner <bohnstinglthomas@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Thomas Ortner
2026-09-09 14:46:31 +01:00
committed by GitHub
co-authored by Harry Mellor
parent 08b3e67b66
commit e509d32b5a
8 changed files with 1035 additions and 90 deletions
@@ -5,13 +5,19 @@
import ast
import types
from abc import ABC, abstractmethod
from collections.abc import Callable
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.transformers.fx_utils import replace_expr
from vllm.model_executor.models.transformers.fx_utils import (
aliasing_names,
bypass_existence_guard,
replace_expr,
self_call_and_refs,
written_names,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
@@ -160,6 +166,53 @@ class StackedFuser(RewriteFuser):
fused layer into its per-shard configs."""
return {self.merged_name: [name for name, _ in self.shards]}
def _unguarded_calls(
self, funcdef: ast.FunctionDef, names: Iterable[str]
) -> list[ast.Call]:
"""One `self.<name>(arg)` call per projection, existence guards folded.
`update_attrs` deletes the projections it merges, so any reference to one beyond
its call site must be a guard on its existence; folding those to their constant
value keeps the rewritten forward off a name that no longer exists. A reference
that is not such a guard raises, so fusion is skipped."""
calls = []
for name in names:
call, refs = self_call_and_refs(funcdef, name)
for ref in refs:
bypass_existence_guard(funcdef, ref, name)
calls.append(call)
return calls
def _check_input_stable(
self,
funcdef: ast.FunctionDef,
module: nn.Module,
calls: list[ast.Call],
block: list[ast.stmt],
indices: list[int],
) -> None:
"""Raise unless hoisting the merged GEMM preserves what it reads.
Fusing moves every projection to one call at `min(indices)`, so the
merged GEMM reads the input once, up front, where the last of `calls`
would have read it later. That holds only if nothing in between changes
the input, and a change need not name it: writing any view that shares
its storage changes it too. Both halves of the check over-approximate,
since a false hit costs a fusion while a miss returns wrong numbers.
"""
arg_names = {
node.id
for node in ast.walk(calls[0].args[0])
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
}
# Protect the names the input reads and anything that may alias them.
tracked = aliasing_names(funcdef, arg_names, module)
# Any of them rebound, written through, or passed to a call that writes
# its argument would leave the hoisted GEMM reading a different value.
region = block[min(indices) : max(indices) + 1]
if tracked & written_names(region):
raise ValueError("projection input is rebound or mutated before all calls")
def _splice_merged_split(
self,
funcdef: ast.FunctionDef,
@@ -137,8 +137,8 @@ class GLUFuser(MergedColumnParallelFuser):
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
funcdef, fn = recover_forward(type(module))
act_call = single_self_call(funcdef, self.act_name)
gate_call = single_self_call(funcdef, self.gate_name)
up_call = single_self_call(funcdef, self.up_name)
calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name))
gate_call, up_call = calls
if act_call.args[0] is not gate_call:
raise ValueError("activation does not directly wrap the gate")
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
@@ -12,11 +12,10 @@ from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
block_chain,
compile_forward,
innermost_block,
is_linear,
recover_forward,
single_self_call,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
@@ -92,30 +91,20 @@ class MergedColumnParallelFuser(StackedFuser):
def update_forward(self, module: nn.Module) -> None:
"""Replace the parallel calls with one merged call and split."""
funcdef, fn = recover_forward(type(module))
calls = [single_self_call(funcdef, name) for name in self.linear_names]
calls = self._unguarded_calls(funcdef, self.linear_names)
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
raise ValueError("parallel linears read different inputs")
blocks = [innermost_block(funcdef.body, call) for call in calls]
if any(found is None for found in blocks):
chains = [block_chain(funcdef.body, call) for call in calls]
if any(not chain for chain in chains):
raise ValueError("parallel linear calls not found in the function body")
blocks = [chain[-1] for chain in chains]
if len(set(id(block) for block, _ in blocks)) != 1:
raise ValueError("parallel linear calls are in different blocks")
block = blocks[0][0]
index = min(index for _, index in blocks)
calls_have_name_arg = all(isinstance(call.args[0], ast.Name) for call in calls)
# Moving projections must not cross operations that can change their input.
for statement in block[index : max(index for _, index in blocks) + 1]:
if not isinstance(statement, (ast.Assign, ast.Return)):
raise ValueError("parallel projections cross other operations")
if isinstance(statement, ast.Assign) and any(
not isinstance(target, ast.Name) for target in statement.targets
):
raise ValueError("parallel projection assignment has side effects")
value = statement.value
values = value.elts if isinstance(value, ast.Tuple) else [value]
if any(value not in calls for value in values) or not calls_have_name_arg:
raise ValueError("parallel projections cross other operations")
indices = [index for _, index in blocks]
index = min(indices)
self._check_input_stable(funcdef, module, calls, block, indices)
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
self._splice_merged_split(funcdef, calls, block, index)
@@ -21,7 +21,6 @@ from vllm.model_executor.models.transformers.fx_utils import (
recover_forward,
replace_expr,
returned_linear,
single_self_call,
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
@@ -239,11 +238,12 @@ class MLAFuser(StackedFuser):
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
# The fused call is inserted at the top-level statement preceding both.
names = [self.q_a_proj_name, self.kv_a_proj_name]
calls = [single_self_call(funcdef, name) for name in names]
calls = self._unguarded_calls(funcdef, names)
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
raise ValueError("down-projections read different inputs")
index = min(_top_level_index(funcdef, call) for call in calls)
self._splice_merged_split(funcdef, calls, funcdef.body, index)
indices = [_top_level_index(funcdef, call) for call in calls]
self._check_input_stable(funcdef, module, calls, funcdef.body, indices)
self._splice_merged_split(funcdef, calls, funcdef.body, min(indices))
# Transformers expands the latent into full key/value in a dedicated method.
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
@@ -15,12 +15,11 @@ from vllm.model_executor.models.transformers.fusers.base import (
fused_head_size,
)
from vllm.model_executor.models.transformers.fx_utils import (
block_chain,
compile_forward,
innermost_block,
is_linear,
recover_forward,
returned_linear,
single_self_call,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
@@ -99,12 +98,18 @@ class QKVFuser(StackedFuser):
return cls(source_cls=type(module).__name__, **names)
def update_forward(self, module: nn.Module) -> None:
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source."""
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source.
A projection may be guarded by an existence check -- a
`self.<proj> is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare
truthiness test -- which is folded to its constant value (see
`bypass_existence_guard`). The calls may sit in different branches, so the
fused GEMM is inserted before the earliest of them, in the innermost block
that dominates all three.
"""
funcdef, fn = recover_forward(type(module))
calls = [
single_self_call(funcdef, name)
for name in (self.q_name, self.k_name, self.v_name)
]
proj_names = (self.q_name, self.k_name, self.v_name)
calls = self._unguarded_calls(funcdef, proj_names)
arg_dumps = {ast.dump(call.args[0]) for call in calls}
if len(arg_dumps) != 1:
raise ValueError("projection inputs are written differently")
@@ -115,7 +120,7 @@ class QKVFuser(StackedFuser):
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
} - {self.q_name, self.k_name, self.v_name}
} - set(proj_names)
for node in ast.walk(funcdef):
if (
isinstance(node, ast.Call)
@@ -124,16 +129,24 @@ class QKVFuser(StackedFuser):
and any(ast.dump(arg) in arg_dumps for arg in node.args)
):
raise ValueError("another linear consumes the same input")
blocks = [innermost_block(funcdef.body, call) for call in calls]
if any(found is None for found in blocks):
# Insert the fused GEMM before the earliest call, in the innermost block
# common to all three (the calls may be split across branches).
chains = [block_chain(funcdef.body, call) for call in calls]
if any(not chain for chain in chains):
raise ValueError("projection calls not found in the function body")
if len({id(block) for block, _ in blocks}) != 1:
raise ValueError("projection calls are in different blocks")
depth = 0
for level in zip(*chains):
if len({id(block) for block, _ in level}) != 1:
break
depth += 1
block = chains[0][depth - 1][0]
indices = [chain[depth - 1][1] for chain in chains]
insert_index = min(indices)
self._check_input_stable(funcdef, module, calls, block, indices)
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
block = blocks[0][0]
index = min(index for _, index in blocks)
self._splice_merged_split(funcdef, calls, block, index)
self._splice_merged_split(funcdef, calls, block, insert_index)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool: