Fix MQA with tensor parallelism on transformers modeling backend (#49987)

Signed-off-by: microslaw <milosz.grunwald@intel.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Miłosz Grunwald
2026-07-27 22:51:17 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 28158b2fc3
commit ebcef33766
7 changed files with 396 additions and 44 deletions
@@ -18,9 +18,10 @@ from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
BaseFuser,
GLUFuser,
PackedQKVFuser,
QKVFuser,
RewriteFuser,
RMSNormFuser,
StackedFuser,
)
from vllm.model_executor.models.transformers.fx_utils import trace
@@ -47,9 +48,9 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
return None
if (graph := trace(module)) is None:
return None
for fuser_cls in (GLUFuser, QKVFuser, RMSNormFuser):
for fuser_cls in (GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser):
if (fuser := fuser_cls.match(graph, module)) is not None:
if isinstance(fuser, StackedFuser):
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
@@ -2,17 +2,24 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.base import BaseFuser, StackedFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
StackedFuser,
)
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"BaseFuser",
"RewriteFuser",
"StackedFuser",
"GLUFuser",
"MoEBlockFuser",
"PackedQKVFuser",
"QKVFuser",
"RMSNormFuser",
]
@@ -57,27 +57,61 @@ class BaseFuser(ABC):
return {}
def local_output_sizes(merged_name: str) -> str:
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
merged = f"self.{merged_name}"
return f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
@dataclass
class StackedFuser(BaseFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it.
class RewriteFuser(BaseFuser):
"""A fuser that rewrites the module's forward and rebinds it.
`match` and `update_forward` analyse the class once; `fuse` builds the merged
submodule and binds the compiled forward on an instance in place, so it keeps
its class and any attribute the fusion does not consume.
`match` and `update_forward` analyse the class once; `fuse` swaps the
submodules and binds the compiled forward on an instance in place, so it
keeps its class and any attribute the fusion does not consume.
"""
merged_name: ClassVar[str]
"""Attribute name of the merged module created by `update_attrs`."""
merged_cls: ClassVar[str]
"""Name of the vLLM class the merged projection becomes (for logging)."""
source_cls: str
"""Class of the HF module the fused projections belonged to (for logging)."""
fused_forward: Callable = field(init=False, repr=False)
"""The compiled rewritten forward, set by `update_forward`."""
@abstractmethod
def update_forward(self, module: nn.Module) -> None:
"""Rewrite and compile `type(module)`'s forward source.
Raises if the source does not admit the rewrite (fusion is then skipped).
"""
@abstractmethod
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace `module`'s submodules with their vLLM equivalents."""
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@dataclass
class StackedFuser(RewriteFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it."""
merged_name: ClassVar[str]
"""Attribute name of the merged module created by `update_attrs`."""
merged_cls: ClassVar[str]
"""Name of the vLLM class the merged projection becomes (for logging)."""
def info(self, name: str) -> str:
sources = " + ".join(shard for shard, _ in self.shards)
return (
@@ -108,26 +142,3 @@ class StackedFuser(BaseFuser):
"""`{merged_name: [projection names]}` so quantization can unpack the
fused layer into its per-shard configs."""
return {self.merged_name: [name for name, _ in self.shards]}
@abstractmethod
def update_forward(self, module: nn.Module) -> None:
"""Rewrite and compile `type(module)`'s forward source.
Raises if the source does not admit the rewrite (fusion is then skipped).
"""
@abstractmethod
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace `module`'s submodules with the merged module."""
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@@ -0,0 +1,163 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
RewriteFuser,
local_output_sizes,
)
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
is_method,
recover_forward,
returned_linear,
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class PackedQKVFuser(RewriteFuser):
"""Fuser for attention with q, k and v packed into one projection."""
qkv_name: str
o_name: str | None
q_size: int
kv_size: int
def info(self, name: str) -> str:
return (
f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear"
)
@staticmethod
def _packed_sizes(node: fx.Node) -> tuple[int, int] | None:
"""`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one."""
if not is_method(node, "split") or len(node.args) < 2:
return None
sizes = node.args[1]
if not isinstance(sizes, (tuple, list)) or len(sizes) != 3:
return None
if not all(isinstance(size, int) for size in sizes):
return None
q_size, k_size, v_size = sizes
if k_size != v_size or q_size < k_size:
return None
return q_size, k_size
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None":
for node in graph.nodes:
if (sizes := cls._packed_sizes(node)) is None:
continue
q_size, kv_size = sizes
qkv_node = upstream_linear(node.args[0], module)
if qkv_node is None:
continue
qkv_name = str(qkv_node.target)
# The split must consume the whole projection.
if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size:
continue
# o_proj produces the module's output and consumes the query width.
o_name = returned_linear(graph, module)
if o_name == qkv_name or (
o_name is not None
and module.get_submodule(o_name).in_features != q_size
):
o_name = None
return cls(
source_cls=type(module).__name__,
qkv_name=qkv_name,
o_name=o_name,
q_size=q_size,
kv_size=kv_size,
)
return None
def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call:
"""The unique `self.<qkv_name>(...)....split((a, b, c), ...)` call."""
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "split"
and node.args
and isinstance(node.args[0], (ast.Tuple, ast.List))
and len(node.args[0].elts) == 3
and any(
isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name
for inner in ast.walk(node.func.value)
)
]
if len(calls) != 1:
raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits")
return calls[0]
def update_forward(self, module: nn.Module) -> None:
"""Rewrite the split sizes to the sharded projection's per-rank widths."""
funcdef, fn = recover_forward(type(module))
split = self._split_call(funcdef)
# (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes]
sections = local_output_sizes(self.qkv_name)
split.args[0] = ast.parse(sections, mode="eval").body
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible with a head-sharded packed GEMM."""
head_size = vllm_config.model_config.get_head_size()
qkv = module.get_submodule(self.qkv_name)
compatible = (
self.q_size % head_size == 0
and self.kv_size % head_size == 0
and qkv.out_features == self.q_size + 2 * self.kv_size
)
if not compatible:
logger.debug("%s is not compatible with packed QKV fusion", type(module))
return compatible
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = vllm_config.model_config.get_head_size()
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
qkv = module.get_submodule(self.qkv_name)
merged = QKVParallelLinear(
hidden_size=qkv.in_features,
head_size=head_size,
total_num_heads=self.q_size // head_size,
total_num_kv_heads=self.kv_size // head_size,
bias=qkv.bias is not None,
quant_config=quant_config,
prefix=qkv_prefix,
return_bias=False,
)
setattr(module, self.qkv_name, merged)
log_replacement(qkv_prefix, qkv, merged)
# If there is an output projection, we know it must be rowwise.
if self.o_name is not None:
o_prefix = maybe_prefix(prefix, self.o_name)
o_proj = module.get_submodule(self.o_name)
new_o = replace_linear_class(
o_proj, "rowwise", quant_config, prefix=o_prefix
)
setattr(module, self.o_name, new_o)
log_replacement(o_prefix, o_proj, new_o)
@@ -10,7 +10,10 @@ from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fusers.base import (
StackedFuser,
local_output_sizes,
)
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
innermost_block,
@@ -134,7 +137,7 @@ class QKVFuser(StackedFuser):
if names & set(temps):
raise ValueError("fused temporaries would shadow existing names")
merged = f"self.{self.merged_name}"
sections = f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
sections = local_output_sizes(self.merged_name)
template = f"{', '.join(temps)} = {merged}(__arg__).split({sections}, -1)"
assign = ast.parse(template).body[0]
arg = next(
@@ -394,8 +394,10 @@ def output_value(graph: fx.Graph) -> object | None:
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
"""Nearest linear producing `node`, walking back through splits/reshapes.
Never walks through a leaf call (e.g. an attention interface): its inputs
are what attention consumes, not what produced the value."""
Non-linear submodules are transparent too (e.g. the dropout GPT-style
attentions apply after their output projection). Never walks through a leaf
call (e.g. an attention interface): its inputs are what attention consumes,
not what produced the value."""
stack = [node]
seen: set[fx.Node] = set()
while stack:
@@ -405,7 +407,11 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
seen.add(current)
if is_linear(current, module):
return current
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
if current.op in (
"call_function",
"call_method",
"call_module",
) and not is_leaf_call(current):
stack.extend(current.args)
return None