[Transformers backend] Enable QKV-Fuser for Gemma4 (#55690)
Signed-off-by: Thomas Ortner <boh@zurich.ibm.com> Signed-off-by: Thomas Ortner <bohnstinglthomas@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
08b3e67b66
commit
e509d32b5a
@@ -5,13 +5,19 @@
|
||||
import ast
|
||||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.model_executor.models.transformers.fx_utils import replace_expr
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
aliasing_names,
|
||||
bypass_existence_guard,
|
||||
replace_expr,
|
||||
self_call_and_refs,
|
||||
written_names,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
@@ -160,6 +166,53 @@ class StackedFuser(RewriteFuser):
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _unguarded_calls(
|
||||
self, funcdef: ast.FunctionDef, names: Iterable[str]
|
||||
) -> list[ast.Call]:
|
||||
"""One `self.<name>(arg)` call per projection, existence guards folded.
|
||||
|
||||
`update_attrs` deletes the projections it merges, so any reference to one beyond
|
||||
its call site must be a guard on its existence; folding those to their constant
|
||||
value keeps the rewritten forward off a name that no longer exists. A reference
|
||||
that is not such a guard raises, so fusion is skipped."""
|
||||
calls = []
|
||||
for name in names:
|
||||
call, refs = self_call_and_refs(funcdef, name)
|
||||
for ref in refs:
|
||||
bypass_existence_guard(funcdef, ref, name)
|
||||
calls.append(call)
|
||||
return calls
|
||||
|
||||
def _check_input_stable(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
module: nn.Module,
|
||||
calls: list[ast.Call],
|
||||
block: list[ast.stmt],
|
||||
indices: list[int],
|
||||
) -> None:
|
||||
"""Raise unless hoisting the merged GEMM preserves what it reads.
|
||||
|
||||
Fusing moves every projection to one call at `min(indices)`, so the
|
||||
merged GEMM reads the input once, up front, where the last of `calls`
|
||||
would have read it later. That holds only if nothing in between changes
|
||||
the input, and a change need not name it: writing any view that shares
|
||||
its storage changes it too. Both halves of the check over-approximate,
|
||||
since a false hit costs a fusion while a miss returns wrong numbers.
|
||||
"""
|
||||
arg_names = {
|
||||
node.id
|
||||
for node in ast.walk(calls[0].args[0])
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
|
||||
}
|
||||
# Protect the names the input reads and anything that may alias them.
|
||||
tracked = aliasing_names(funcdef, arg_names, module)
|
||||
# Any of them rebound, written through, or passed to a call that writes
|
||||
# its argument would leave the hoisted GEMM reading a different value.
|
||||
region = block[min(indices) : max(indices) + 1]
|
||||
if tracked & written_names(region):
|
||||
raise ValueError("projection input is rebound or mutated before all calls")
|
||||
|
||||
def _splice_merged_split(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
|
||||
@@ -137,8 +137,8 @@ class GLUFuser(MergedColumnParallelFuser):
|
||||
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
act_call = single_self_call(funcdef, self.act_name)
|
||||
gate_call = single_self_call(funcdef, self.gate_name)
|
||||
up_call = single_self_call(funcdef, self.up_name)
|
||||
calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name))
|
||||
gate_call, up_call = calls
|
||||
if act_call.args[0] is not gate_call:
|
||||
raise ValueError("activation does not directly wrap the gate")
|
||||
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
|
||||
|
||||
@@ -12,11 +12,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
block_chain,
|
||||
compile_forward,
|
||||
innermost_block,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
@@ -92,30 +91,20 @@ class MergedColumnParallelFuser(StackedFuser):
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace the parallel calls with one merged call and split."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
calls = [single_self_call(funcdef, name) for name in self.linear_names]
|
||||
calls = self._unguarded_calls(funcdef, self.linear_names)
|
||||
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
|
||||
raise ValueError("parallel linears read different inputs")
|
||||
blocks = [innermost_block(funcdef.body, call) for call in calls]
|
||||
if any(found is None for found in blocks):
|
||||
chains = [block_chain(funcdef.body, call) for call in calls]
|
||||
if any(not chain for chain in chains):
|
||||
raise ValueError("parallel linear calls not found in the function body")
|
||||
blocks = [chain[-1] for chain in chains]
|
||||
if len(set(id(block) for block, _ in blocks)) != 1:
|
||||
raise ValueError("parallel linear calls are in different blocks")
|
||||
|
||||
block = blocks[0][0]
|
||||
index = min(index for _, index in blocks)
|
||||
calls_have_name_arg = all(isinstance(call.args[0], ast.Name) for call in calls)
|
||||
# Moving projections must not cross operations that can change their input.
|
||||
for statement in block[index : max(index for _, index in blocks) + 1]:
|
||||
if not isinstance(statement, (ast.Assign, ast.Return)):
|
||||
raise ValueError("parallel projections cross other operations")
|
||||
if isinstance(statement, ast.Assign) and any(
|
||||
not isinstance(target, ast.Name) for target in statement.targets
|
||||
):
|
||||
raise ValueError("parallel projection assignment has side effects")
|
||||
value = statement.value
|
||||
values = value.elts if isinstance(value, ast.Tuple) else [value]
|
||||
if any(value not in calls for value in values) or not calls_have_name_arg:
|
||||
raise ValueError("parallel projections cross other operations")
|
||||
indices = [index for _, index in blocks]
|
||||
index = min(indices)
|
||||
self._check_input_stable(funcdef, module, calls, block, indices)
|
||||
|
||||
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
|
||||
self._splice_merged_split(funcdef, calls, block, index)
|
||||
|
||||
@@ -21,7 +21,6 @@ from vllm.model_executor.models.transformers.fx_utils import (
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
single_self_call,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
@@ -239,11 +238,12 @@ class MLAFuser(StackedFuser):
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
# The fused call is inserted at the top-level statement preceding both.
|
||||
names = [self.q_a_proj_name, self.kv_a_proj_name]
|
||||
calls = [single_self_call(funcdef, name) for name in names]
|
||||
calls = self._unguarded_calls(funcdef, names)
|
||||
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
|
||||
raise ValueError("down-projections read different inputs")
|
||||
index = min(_top_level_index(funcdef, call) for call in calls)
|
||||
self._splice_merged_split(funcdef, calls, funcdef.body, index)
|
||||
indices = [_top_level_index(funcdef, call) for call in calls]
|
||||
self._check_input_stable(funcdef, module, calls, funcdef.body, indices)
|
||||
self._splice_merged_split(funcdef, calls, funcdef.body, min(indices))
|
||||
|
||||
# Transformers expands the latent into full key/value in a dedicated method.
|
||||
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
|
||||
|
||||
@@ -15,12 +15,11 @@ from vllm.model_executor.models.transformers.fusers.base import (
|
||||
fused_head_size,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
block_chain,
|
||||
compile_forward,
|
||||
innermost_block,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
returned_linear,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
@@ -99,12 +98,18 @@ class QKVFuser(StackedFuser):
|
||||
return cls(source_cls=type(module).__name__, **names)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source."""
|
||||
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source.
|
||||
|
||||
A projection may be guarded by an existence check -- a
|
||||
`self.<proj> is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare
|
||||
truthiness test -- which is folded to its constant value (see
|
||||
`bypass_existence_guard`). The calls may sit in different branches, so the
|
||||
fused GEMM is inserted before the earliest of them, in the innermost block
|
||||
that dominates all three.
|
||||
"""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
calls = [
|
||||
single_self_call(funcdef, name)
|
||||
for name in (self.q_name, self.k_name, self.v_name)
|
||||
]
|
||||
proj_names = (self.q_name, self.k_name, self.v_name)
|
||||
calls = self._unguarded_calls(funcdef, proj_names)
|
||||
arg_dumps = {ast.dump(call.args[0]) for call in calls}
|
||||
if len(arg_dumps) != 1:
|
||||
raise ValueError("projection inputs are written differently")
|
||||
@@ -115,7 +120,7 @@ class QKVFuser(StackedFuser):
|
||||
name
|
||||
for name, child in module.named_children()
|
||||
if isinstance(child, nn.Linear)
|
||||
} - {self.q_name, self.k_name, self.v_name}
|
||||
} - set(proj_names)
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
@@ -124,16 +129,24 @@ class QKVFuser(StackedFuser):
|
||||
and any(ast.dump(arg) in arg_dumps for arg in node.args)
|
||||
):
|
||||
raise ValueError("another linear consumes the same input")
|
||||
blocks = [innermost_block(funcdef.body, call) for call in calls]
|
||||
if any(found is None for found in blocks):
|
||||
|
||||
# Insert the fused GEMM before the earliest call, in the innermost block
|
||||
# common to all three (the calls may be split across branches).
|
||||
chains = [block_chain(funcdef.body, call) for call in calls]
|
||||
if any(not chain for chain in chains):
|
||||
raise ValueError("projection calls not found in the function body")
|
||||
if len({id(block) for block, _ in blocks}) != 1:
|
||||
raise ValueError("projection calls are in different blocks")
|
||||
depth = 0
|
||||
for level in zip(*chains):
|
||||
if len({id(block) for block, _ in level}) != 1:
|
||||
break
|
||||
depth += 1
|
||||
block = chains[0][depth - 1][0]
|
||||
indices = [chain[depth - 1][1] for chain in chains]
|
||||
insert_index = min(indices)
|
||||
self._check_input_stable(funcdef, module, calls, block, indices)
|
||||
|
||||
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
|
||||
block = blocks[0][0]
|
||||
index = min(index for _, index in blocks)
|
||||
self._splice_merged_split(funcdef, calls, block, index)
|
||||
self._splice_merged_split(funcdef, calls, block, insert_index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
|
||||
Reference in New Issue
Block a user