Fix MQA with tensor parallelism on transformers modeling backend (#49987)
Signed-off-by: microslaw <milosz.grunwald@intel.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
28158b2fc3
commit
ebcef33766
@@ -18,9 +18,10 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers import (
|
||||
BaseFuser,
|
||||
GLUFuser,
|
||||
PackedQKVFuser,
|
||||
QKVFuser,
|
||||
RewriteFuser,
|
||||
RMSNormFuser,
|
||||
StackedFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
@@ -47,9 +48,9 @@ def get_fuser(module: nn.Module) -> BaseFuser | None:
|
||||
return None
|
||||
if (graph := trace(module)) is None:
|
||||
return None
|
||||
for fuser_cls in (GLUFuser, QKVFuser, RMSNormFuser):
|
||||
for fuser_cls in (GLUFuser, QKVFuser, PackedQKVFuser, RMSNormFuser):
|
||||
if (fuser := fuser_cls.match(graph, module)) is not None:
|
||||
if isinstance(fuser, StackedFuser):
|
||||
if isinstance(fuser, RewriteFuser):
|
||||
try:
|
||||
fuser.update_forward(module)
|
||||
except Exception as exc:
|
||||
|
||||
@@ -2,17 +2,24 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Concrete fusers for the Transformers modeling backend."""
|
||||
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser, StackedFuser
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
BaseFuser,
|
||||
RewriteFuser,
|
||||
StackedFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
|
||||
|
||||
__all__ = [
|
||||
"BaseFuser",
|
||||
"RewriteFuser",
|
||||
"StackedFuser",
|
||||
"GLUFuser",
|
||||
"MoEBlockFuser",
|
||||
"PackedQKVFuser",
|
||||
"QKVFuser",
|
||||
"RMSNormFuser",
|
||||
]
|
||||
|
||||
@@ -57,27 +57,61 @@ class BaseFuser(ABC):
|
||||
return {}
|
||||
|
||||
|
||||
def local_output_sizes(merged_name: str) -> str:
|
||||
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
|
||||
merged = f"self.{merged_name}"
|
||||
return f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackedFuser(BaseFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it.
|
||||
class RewriteFuser(BaseFuser):
|
||||
"""A fuser that rewrites the module's forward and rebinds it.
|
||||
|
||||
`match` and `update_forward` analyse the class once; `fuse` builds the merged
|
||||
submodule and binds the compiled forward on an instance in place, so it keeps
|
||||
its class and any attribute the fusion does not consume.
|
||||
`match` and `update_forward` analyse the class once; `fuse` swaps the
|
||||
submodules and binds the compiled forward on an instance in place, so it
|
||||
keeps its class and any attribute the fusion does not consume.
|
||||
"""
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
merged_cls: ClassVar[str]
|
||||
"""Name of the vLLM class the merged projection becomes (for logging)."""
|
||||
|
||||
source_cls: str
|
||||
"""Class of the HF module the fused projections belonged to (for logging)."""
|
||||
|
||||
fused_forward: Callable = field(init=False, repr=False)
|
||||
"""The compiled rewritten forward, set by `update_forward`."""
|
||||
|
||||
@abstractmethod
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite and compile `type(module)`'s forward source.
|
||||
|
||||
Raises if the source does not admit the rewrite (fusion is then skipped).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace `module`'s submodules with their vLLM equivalents."""
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it."""
|
||||
|
||||
merged_name: ClassVar[str]
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
merged_cls: ClassVar[str]
|
||||
"""Name of the vLLM class the merged projection becomes (for logging)."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
sources = " + ".join(shard for shard, _ in self.shards)
|
||||
return (
|
||||
@@ -108,26 +142,3 @@ class StackedFuser(BaseFuser):
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
@abstractmethod
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite and compile `type(module)`'s forward source.
|
||||
|
||||
Raises if the source does not admit the rewrite (fusion is then skipped).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace `module`'s submodules with the merged module."""
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
|
||||
@@ -0,0 +1,163 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
RewriteFuser,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
is_method,
|
||||
recover_forward,
|
||||
returned_linear,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedQKVFuser(RewriteFuser):
|
||||
"""Fuser for attention with q, k and v packed into one projection."""
|
||||
|
||||
qkv_name: str
|
||||
o_name: str | None
|
||||
q_size: int
|
||||
kv_size: int
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
return (
|
||||
f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _packed_sizes(node: fx.Node) -> tuple[int, int] | None:
|
||||
"""`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one."""
|
||||
if not is_method(node, "split") or len(node.args) < 2:
|
||||
return None
|
||||
sizes = node.args[1]
|
||||
if not isinstance(sizes, (tuple, list)) or len(sizes) != 3:
|
||||
return None
|
||||
if not all(isinstance(size, int) for size in sizes):
|
||||
return None
|
||||
q_size, k_size, v_size = sizes
|
||||
if k_size != v_size or q_size < k_size:
|
||||
return None
|
||||
return q_size, k_size
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None":
|
||||
for node in graph.nodes:
|
||||
if (sizes := cls._packed_sizes(node)) is None:
|
||||
continue
|
||||
q_size, kv_size = sizes
|
||||
qkv_node = upstream_linear(node.args[0], module)
|
||||
if qkv_node is None:
|
||||
continue
|
||||
qkv_name = str(qkv_node.target)
|
||||
# The split must consume the whole projection.
|
||||
if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size:
|
||||
continue
|
||||
# o_proj produces the module's output and consumes the query width.
|
||||
o_name = returned_linear(graph, module)
|
||||
if o_name == qkv_name or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features != q_size
|
||||
):
|
||||
o_name = None
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
qkv_name=qkv_name,
|
||||
o_name=o_name,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
)
|
||||
return None
|
||||
|
||||
def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call:
|
||||
"""The unique `self.<qkv_name>(...)....split((a, b, c), ...)` call."""
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "split"
|
||||
and node.args
|
||||
and isinstance(node.args[0], (ast.Tuple, ast.List))
|
||||
and len(node.args[0].elts) == 3
|
||||
and any(
|
||||
isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name
|
||||
for inner in ast.walk(node.func.value)
|
||||
)
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits")
|
||||
return calls[0]
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite the split sizes to the sharded projection's per-rank widths."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
split = self._split_call(funcdef)
|
||||
# (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes]
|
||||
sections = local_output_sizes(self.qkv_name)
|
||||
split.args[0] = ast.parse(sections, mode="eval").body
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible with a head-sharded packed GEMM."""
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
compatible = (
|
||||
self.q_size % head_size == 0
|
||||
and self.kv_size % head_size == 0
|
||||
and qkv.out_features == self.q_size + 2 * self.kv_size
|
||||
)
|
||||
if not compatible:
|
||||
logger.debug("%s is not compatible with packed QKV fusion", type(module))
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = vllm_config.model_config.get_head_size()
|
||||
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
merged = QKVParallelLinear(
|
||||
hidden_size=qkv.in_features,
|
||||
head_size=head_size,
|
||||
total_num_heads=self.q_size // head_size,
|
||||
total_num_kv_heads=self.kv_size // head_size,
|
||||
bias=qkv.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=qkv_prefix,
|
||||
return_bias=False,
|
||||
)
|
||||
setattr(module, self.qkv_name, merged)
|
||||
log_replacement(qkv_prefix, qkv, merged)
|
||||
# If there is an output projection, we know it must be rowwise.
|
||||
if self.o_name is not None:
|
||||
o_prefix = maybe_prefix(prefix, self.o_name)
|
||||
o_proj = module.get_submodule(self.o_name)
|
||||
new_o = replace_linear_class(
|
||||
o_proj, "rowwise", quant_config, prefix=o_prefix
|
||||
)
|
||||
setattr(module, self.o_name, new_o)
|
||||
log_replacement(o_prefix, o_proj, new_o)
|
||||
@@ -10,7 +10,10 @@ from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
innermost_block,
|
||||
@@ -134,7 +137,7 @@ class QKVFuser(StackedFuser):
|
||||
if names & set(temps):
|
||||
raise ValueError("fused temporaries would shadow existing names")
|
||||
merged = f"self.{self.merged_name}"
|
||||
sections = f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
|
||||
sections = local_output_sizes(self.merged_name)
|
||||
template = f"{', '.join(temps)} = {merged}(__arg__).split({sections}, -1)"
|
||||
assign = ast.parse(template).body[0]
|
||||
arg = next(
|
||||
|
||||
@@ -394,8 +394,10 @@ def output_value(graph: fx.Graph) -> object | None:
|
||||
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear producing `node`, walking back through splits/reshapes.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): its inputs
|
||||
are what attention consumes, not what produced the value."""
|
||||
Non-linear submodules are transparent too (e.g. the dropout GPT-style
|
||||
attentions apply after their output projection). Never walks through a leaf
|
||||
call (e.g. an attention interface): its inputs are what attention consumes,
|
||||
not what produced the value."""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
@@ -405,7 +407,11 @@ def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
seen.add(current)
|
||||
if is_linear(current, module):
|
||||
return current
|
||||
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
|
||||
if current.op in (
|
||||
"call_function",
|
||||
"call_method",
|
||||
"call_module",
|
||||
) and not is_leaf_call(current):
|
||||
stack.extend(current.args)
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user