Make the Transformers modeling backend as fast as native vLLM (#47187)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2026-07-06 16:59:14 +01:00
committed by GitHub
co-authored by mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent 5ad11172b7
commit 5bce653e09
23 changed files with 2679 additions and 160 deletions
@@ -16,6 +16,10 @@
# limitations under the License.
"""Wrapper around `transformers` models"""
from typing import TYPE_CHECKING
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import Base
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.legacy import LegacyMixin
@@ -32,6 +36,36 @@ from vllm.model_executor.models.transformers.pooling import (
)
from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.attention import Attention
def vllm_attention_forward(
# Transformers args
module: "torch.nn.Module",
query: "torch.Tensor",
key: "torch.Tensor",
value: "torch.Tensor",
attention_mask: "torch.Tensor",
# Transformers kwargs
scaling: float | None = None,
# vLLM kwargs
attention_instances: dict[int, "Attention"] | None = None,
**kwargs,
):
self_attn = attention_instances[module.layer_idx]
if scaling is not None:
self_attn.impl.scale = float(scaling)
hidden = query.shape[-2]
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
return self_attn.forward(query, key, value), None
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_attention_forward
# Text only models
class TransformersForCausalLM(CausalMixin, Base): ...
+65 -53
View File
@@ -31,7 +31,6 @@ from transformers.conversion_mapping import (
WeightRenaming,
get_model_conversion_mapping,
)
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.compilation.decorators import support_torch_compile
from vllm.config.utils import getattr_iter
@@ -42,6 +41,7 @@ from vllm.model_executor.layers.attention import (
Attention,
EncoderOnlyAttention,
)
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.interfaces import (
SupportsEagle,
@@ -51,6 +51,7 @@ from vllm.model_executor.models.interfaces import (
SupportsQuant,
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.utils import (
can_enable_torch_compile,
get_feature_request_tip,
@@ -58,7 +59,6 @@ from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_conv_class,
replace_linear_class,
replace_rms_norm_class,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
@@ -74,37 +74,10 @@ if TYPE_CHECKING:
from transformers import PreTrainedModel
from vllm.config import VllmConfig
else:
PreTrainedModel = object
logger = init_logger(__name__)
def vllm_flash_attention_forward(
# Transformers args
module: torch.nn.Module,
query: torch.Tensor,
key: torch.Tensor,
value: torch.Tensor,
attention_mask: torch.Tensor,
# Transformers kwargs
scaling: float | None = None,
# vLLM kwargs
attention_instances: dict[int, Attention] | None = None,
**kwargs,
):
self_attn = attention_instances[module.layer_idx]
if scaling is not None:
self_attn.impl.scale = float(scaling)
hidden = query.shape[-2]
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
return self_attn.forward(query, key, value), None
ALL_ATTENTION_FUNCTIONS["vllm"] = vllm_flash_attention_forward
class Base(
nn.Module,
VllmModel,
@@ -141,6 +114,9 @@ class Base(
"""Ignore unexpected weights whose qualname starts with these prefixes."""
self.ignore_unexpected_suffixes: list[str] = []
"""Ignore unexpected weights whose qualname ends with these suffixes."""
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
@@ -217,7 +193,7 @@ class Base(
self.text_config._attn_implementation = "vllm"
self.config.dtype = torch.get_default_dtype()
def _get_decoder_cls(self, **kwargs: dict) -> type[PreTrainedModel]:
def _get_decoder_cls(self, **kwargs: dict) -> type["PreTrainedModel"]:
"""
Get the decoder class from the model.
@@ -236,7 +212,7 @@ class Base(
def _decorate_cls_for_torch_compile(
self,
cls: type[PreTrainedModel],
cls: type["PreTrainedModel"],
dynamic_arg_dims: dict[str, int] | None,
enable_if: Callable[["VllmConfig"], bool],
is_encoder: bool,
@@ -356,12 +332,24 @@ class Base(
if self.pp_group.world_size <= 1:
return
if not self.model.supports_pp_plan:
if self.model.supports_pp_plan:
module = self.model
names = list(module._pp_plan.keys())
else:
module = self.model.get_decoder()
has_parameters = lambda m: next(m.parameters(), None) is not None
names = [n for n, c in module.named_children() if has_parameters(c)]
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
raise ValueError(
f"{type(self.model)} does not support pipeline parallel. {tip}"
logger.warning(
"%s does not define a pipeline parallel plan. The Transformers "
"modeling backend will infer the split from the layers of %s in order "
"of declaration and keep parameter-free modules on every rank. This "
"may fail if the model's structure is non-standard. %s",
type(self.model),
type(module),
tip,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
@@ -376,10 +364,9 @@ class Base(
module_lists = []
module_list_idx = None
pp_plan = list(self.model._pp_plan.keys())
for i, name in enumerate(pp_plan):
for i, name in enumerate(names):
# attrgetter in case the module is nested (e.g. "text_model.layers")
if isinstance(attrgetter(name)(self.model), nn.ModuleList):
if isinstance(attrgetter(name)(module), nn.ModuleList):
module_lists.append(name)
module_list_idx = i
@@ -389,16 +376,16 @@ class Base(
"in the base model are not supported yet!"
)
if module_list_idx is None:
raise ValueError(f"Could not find `ModuleList` in {type(self.model)}")
raise ValueError(f"Could not find `ModuleList` in {type(module)}")
# Layers before module list
for name in pp_plan[:module_list_idx]:
for name in names[:module_list_idx]:
if self.pp_group.is_first_rank or (
self._get_tie_word_embeddings() and self.pp_group.is_last_rank
):
continue
# attrsetter in case the module is nested (e.g. "text_model.embed_tokens")
attrsetter(name)(self.model, PPMissingLayer())
attrsetter(name)(module, PPMissingLayer())
# Module list
start_layer, end_layer = get_pp_indices(
@@ -406,41 +393,61 @@ class Base(
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
layers_name = pp_plan[module_list_idx]
layers_name = names[module_list_idx]
# attrgetter in case the module is nested (e.g. "text_model.layers")
layers = attrgetter(layers_name)(self.model)
layers = attrgetter(layers_name)(module)
for i in range(len(layers)):
if start_layer <= i and i < end_layer:
continue
layers[i] = PPMissingLayer()
# Layers after module list
for name in pp_plan[module_list_idx + 1 :]:
for name in names[module_list_idx + 1 :]:
# Modules that should be on last rank
if not self.pp_group.is_last_rank:
# attrsetter in case the module is nested (e.g. "text_model.norm")
attrsetter(name)(self.model, PPMissingLayer())
attrsetter(name)(module, PPMissingLayer())
def recursive_replace(self):
"""Recursively replace modules in the model as needed.
Currently, this replaces:
- GLUs with a fused `MergedColumnParallelLinear` + `...AndMul`
- Attention QKV projections with a fused `QKVParallelLinear` + split
- `nn.Linear` with vLLM's tensor parallel linear classes
- `*RMSNorm` with vLLM's `RMSNorm`
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
"""
tp_plan = self.model.tp_plan
tp_plan = self.model.tp_plan or {}
if not tp_plan and self.tp_group.world_size > 1:
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
raise ValueError(
f"{type(self.model)} does not support tensor parallel. {tip}"
logger.warning_once(
"%s does not define a tensor parallel plan. The Transformers modeling "
"backend will shard the model the best it can during graph fusion and "
"replicate the rest. This may be suboptimal or fail if the model does "
"not fuse cleanly. %s",
type(self.model),
tip,
)
# Prefix the patterns because we always start from `self.model`
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
# Detect fusable patterns once per module class (cached, so this is cheap)
fusers = Fusers(self.model, self.model_config)
def register_fusion(fuser: BaseFuser, prefix: str):
"""Register a fused layer's mappings just before it is built."""
orig_to_new_stacked = fuser.orig_to_new_stacked(prefix)
self.hf_to_vllm_mapper.orig_to_new_stacked.update(orig_to_new_stacked)
packed_modules_mapping = fuser.packed_modules_mapping
self.packed_modules_mapping.update(packed_modules_mapping)
if self.quant_config is not None:
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
@@ -479,11 +486,16 @@ class Base(
)
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
new_module = replace_conv_class(child_module)
elif child_module.__class__.__name__.endswith("RMSNorm"):
new_module = replace_rms_norm_class(
child_module, self.text_config.hidden_size
elif (fuser := fusers[child_module]) is not None:
register_fusion(fuser, qual_name)
new_module = fuser.fuse(
child_module, qual_name, self.model_config, self.quant_config
)
else:
logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
# MoERunner can contain aliases of shared experts and gates,
# so we don't want to recurse into it and break weight loading.
_recursive_replace(child_module, prefix=qual_name)
if new_module is not child_module:
@@ -538,7 +550,7 @@ class Base(
num_heads=num_heads,
head_size=head_size,
# NOTE: We use Llama scale as default, if it's set by
# Transformers, it's updated in vllm_flash_attention_forward
# Transformers, it's updated in vllm_attention_forward
scale=head_size**-0.5,
num_kv_heads=num_kv_heads,
cache_config=self.cache_config,
@@ -0,0 +1,78 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser detection for the Transformers modeling backend.
`get_fuser` traces a module class once (see `fx_utils`) and matches it against
each concrete fuser in `fusers`; `Fusers` caches the result per class for a
whole model. `base.recursive_replace` then applies the matched fuser per
instance. RMSNorm-shaped modules the tracer cannot match are warned about.
"""
from collections import UserDict
from typing import TYPE_CHECKING
from cachetools import cached
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
BaseFuser,
GLUFuser,
QKVFuser,
RMSNormFuser,
StackedFuser,
)
from vllm.model_executor.models.transformers.fx_utils import trace
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
logger = init_logger(__name__)
@cached(cache={}, key=type)
def get_fuser(module: nn.Module) -> BaseFuser | None:
"""The fuser for `type(module)` (cached per class), or `None` if no match."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
is_leaf = next(module.children(), None) is None
if n_linear < 2 and not is_leaf:
return None
if (graph := trace(module)) is None:
return None
for fuser_cls in (GLUFuser, QKVFuser, RMSNormFuser):
if (fuser := fuser_cls.match(graph, module)) is not None:
if isinstance(fuser, StackedFuser):
try:
fuser.update_forward(module)
except Exception as exc:
# An unrecognised source just means we cannot fuse here.
logger.debug(
"Could not rewrite %s for fusion: %s", type(module), exc
)
return None
return fuser
# A norm we could not match structurally is left unfused; flag likely misses.
if module.__class__.__name__.endswith("RMSNorm"):
logger.warning_once(
"%s looks like an RMSNorm but its computation did not match the "
"expected pattern, so it was left unfused.",
module.__class__.__name__,
)
return None
class Fusers(UserDict):
"""Mapping from module class to fuser, for all fusable classes in a model."""
def __init__(self, model: nn.Module, model_config: "ModelConfig"):
self.model_config = model_config
super().__init__({type(m): get_fuser(m) for m in model.modules()})
def __getitem__(self, m: nn.Module) -> BaseFuser | None:
fuser = self.data.get(type(m))
if fuser is not None and fuser.validate(m, self.model_config):
return fuser
return None
@@ -0,0 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.base import BaseFuser, StackedFuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"BaseFuser",
"StackedFuser",
"GLUFuser",
"MoEBlockFuser",
"QKVFuser",
"RMSNormFuser",
]
@@ -0,0 +1,146 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base classes for the Transformers backend fusers."""
import types
from abc import ABC, abstractmethod
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
@dataclass
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
`match` analyses the module *class* once (cached, see `get_fuser`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
@classmethod
@abstractmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "BaseFuser | None":
"""Match the pattern in `graph`, returning a fuser if found."""
@abstractmethod
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
"""Whether this fuser can be applied to this `module` instance."""
@abstractmethod
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built)."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
(none unless it stacks weights)."""
return {}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`packed_modules_mapping` entries this fuser contributes (none unless
it stacks weights)."""
return {}
@dataclass
class StackedFuser(BaseFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it.
`match` and `update_forward` analyse the class once; `fuse` builds the merged
submodule and binds the compiled forward on an instance in place, so it keeps
its class and any attribute the fusion does not consume.
"""
merged_name: ClassVar[str]
"""Attribute name of the merged module created by `update_attrs`."""
merged_cls: ClassVar[str]
"""Name of the vLLM class the merged projection becomes (for logging)."""
source_cls: str
"""Class of the HF module the fused projections belonged to (for logging)."""
fused_forward: Callable = field(init=False, repr=False)
"""The compiled rewritten forward, set by `update_forward`."""
def info(self, name: str) -> str:
sources = " + ".join(shard for shard, _ in self.shards)
return (
f"Fused: {sources} ({name}: {self.source_cls}) -> "
f"{self.merged_name} ({self.merged_cls})"
)
@property
@abstractmethod
def shards(self) -> list[tuple[str, ShardId]]:
"""Each projection's original name and its shard id in the merged module.
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
so only this exact layer is remapped, never a same-named projection
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
merged = maybe_prefix(prefix, self.merged_name)
return {
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`{merged_name: [projection names]}` so quantization can unpack the
fused layer into its per-shard configs."""
return {self.merged_name: [name for name, _ in self.shards]}
@abstractmethod
def update_forward(self, module: nn.Module) -> None:
"""Rewrite and compile `type(module)`'s forward source.
Raises if the source does not admit the rewrite (fusion is then skipped).
"""
@abstractmethod
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
) -> None:
"""Replace `module`'s submodules with the merged module."""
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, model_config, quant_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@@ -0,0 +1,218 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLU projection fuser: `act(gate(x)) * up(x)` -> a fused gate/up linear."""
import ast
import operator
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from transformers.activations import ACT2CLS
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import (
_ACTIVATION_AND_MUL_REGISTRY,
get_act_and_mul_fn,
)
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
find_node,
is_linear,
peel,
recover_forward,
replace_expr,
single_self_call,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
CLS2ACT: dict[type, list[str]] = {}
for _act_name, _act_cls in ACT2CLS.items():
if isinstance(_act_cls, tuple):
_act_cls = _act_cls[0]
CLS2ACT.setdefault(_act_cls, []).append(_act_name)
ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys())
@dataclass
class GLUFuser(StackedFuser):
"""Fuser for the GLU pattern `act(gate(x)) * up(x)`."""
act_name: str
gate_name: str
up_name: str
down_name: str | None
merged_name: ClassVar[str] = "gate_up_proj"
merged_cls: ClassVar[str] = "MergedColumnParallelLinear"
@property
def shards(self) -> list[tuple[str, ShardId]]:
return [(self.gate_name, 0), (self.up_name, 1)]
@classmethod
def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool:
"""Is node `act(gate(x))` where `gate` is linear and `act` is not linear."""
return (
node.op == "call_module"
and not is_linear(node, module)
and len(node.args) == 1
and isinstance(node.args[0], fx.Node)
and is_linear(node.args[0], module)
)
@classmethod
def _get_glu_nodes(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node, fx.Node] | None:
"""Search graph for the GLU pattern `act(gate(x)) * up(x)`."""
for mul in graph.nodes:
if (
mul.op == "call_function"
and mul.target == operator.mul
and len(mul.args) == 2
and all(isinstance(arg, fx.Node) for arg in mul.args)
):
a, b = mul.args
if cls._is_act_of_gate(a, module) and is_linear(b, module):
act, gate, up = a, a.args[0], b
elif cls._is_act_of_gate(b, module) and is_linear(a, module):
act, gate, up = b, b.args[0], a
else:
continue
if (
all(len(args) == 1 for args in (gate.args, up.args))
and isinstance(x := gate.args[0], fx.Node)
and x is up.args[0]
):
return act, gate, up, mul
return None
@staticmethod
def _get_act_and_mul_name(act: nn.Module) -> str | None:
"""Get the name of `act` if it has an `...AndMul` equivalent."""
for name in CLS2ACT.get(type(act), []):
if name in ACT_AND_MUL_NAMES:
return name
# nn.GELU is not in ACT2CLS, but could be in model code
if type(act) is nn.GELU:
return "gelu_pytorch_tanh" if act.approximate == "tanh" else "gelu"
return None
@classmethod
def _get_act_and_mul(cls, act: nn.Module) -> nn.Module:
"""Get the `...AndMul` equivalent of a Transformers activation module."""
if name := cls._get_act_and_mul_name(act):
return get_act_and_mul_fn(name)
raise ValueError(f"No AndMul equivalent for {type(act)}")
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "GLUFuser | None":
if (glu_nodes := cls._get_glu_nodes(graph, module)) is None:
return None
act_node, gate_node, up_node, mul_node = glu_nodes
gate = module.get_submodule(gate_node.target)
up = module.get_submodule(up_node.target)
# Shapes must be compatible for a single merged GEMM.
if gate.in_features == up.in_features and (gate.bias is None) == (
up.bias is None
):
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
down_node = find_node(graph, predicate)
return cls(
source_cls=type(module).__name__,
act_name=act_node.target,
gate_name=gate_node.target,
up_name=up_node.target,
down_name=down_node.target if down_node is not None else None,
)
return None
def update_forward(self, module: nn.Module) -> None:
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
funcdef, fn = recover_forward(type(module))
act_call = single_self_call(funcdef, self.act_name)
gate_call = single_self_call(funcdef, self.gate_name)
up_call = single_self_call(funcdef, self.up_name)
if act_call.args[0] is not gate_call:
raise ValueError("activation does not directly wrap the gate")
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
raise ValueError("gate and up inputs are written differently")
muls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.BinOp)
and isinstance(node.op, ast.Mult)
and {id(node.left), id(node.right)} == {id(act_call), id(up_call)}
]
if len(muls) != 1:
raise ValueError("no multiply of the activation and up projection")
# act(gate(x)) * up(x) -> act(gate_up(x))
assert isinstance(gate_call.func, ast.Attribute)
gate_call.func.attr = self.merged_name
replace_expr(funcdef, muls[0], act_call)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
act = module.get_submodule(self.act_name)
if self._get_act_and_mul_name(act) is None:
logger.debug("No AndMul equivalent for %s; skipping fusion", type(act))
return False
return True
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
) -> None:
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
gate = module.get_submodule(self.gate_name)
up = module.get_submodule(self.up_name)
merged = MergedColumnParallelLinear(
input_size=gate.in_features,
output_sizes=[gate.out_features, up.out_features],
bias=gate.bias is not None,
quant_config=quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
)
logger.debug(
"%s: %s, %s: %s -> %s: %s",
self.gate_name,
gate,
self.up_name,
up,
self.merged_name,
merged,
)
setattr(module, self.merged_name, merged)
setattr(module, self.act_name, act_fn)
# Drop the consumed submodules so their (meta) params are not expected.
delattr(module, self.gate_name)
delattr(module, self.up_name)
# If there is a down projection, we know it must be rowwise.
if self.down_name is not None:
down_prefix = maybe_prefix(prefix, self.down_name)
down = module.get_submodule(self.down_name)
new_down = replace_linear_class(
down, "rowwise", quant_config, prefix=down_prefix
)
setattr(module, self.down_name, new_down)
log_replacement(down_prefix, down, new_down)
@@ -0,0 +1,268 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE fuser: route an HF MoE block through `FusedMoE` with vLLM's own routing."""
import ast
import inspect
import textwrap
import types
from collections.abc import Iterator
from dataclasses import dataclass
from itertools import chain
import torch
from torch import fx, nn
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
is_op,
peel,
trace,
)
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
stack = list(ast.iter_child_nodes(node))
while stack:
child = stack.pop()
if isinstance(child, ast.Return):
yield child
elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
stack.extend(ast.iter_child_nodes(child))
def _returns_tuple(cls: type[nn.Module]) -> bool:
"""Does `cls.forward()` return a tuple?"""
try:
source = textwrap.dedent(inspect.getsource(inspect.unwrap(cls.forward)))
forward = ast.parse(source).body[0]
except (OSError, SyntaxError, TypeError, IndexError):
return True
# Names bound to a tuple literal, e.g. `out = hidden, logits` then `return out`.
tuple_names = {
target.id
for node in ast.walk(forward)
if isinstance(node, ast.Assign) and isinstance(node.value, ast.Tuple)
for target in node.targets
if isinstance(target, ast.Name)
}
def yields_tuple(value: ast.expr | None) -> bool:
if isinstance(value, ast.Tuple):
return True
if isinstance(value, ast.Name):
return value.id in tuple_names
if isinstance(value, ast.IfExp):
return yields_tuple(value.body) or yields_tuple(value.orelse)
return False
return any(yields_tuple(ret.value) for ret in _own_returns(forward))
def _is_scalar_gate(module: nn.Module) -> bool:
"""A linear projecting to a single logit (the shared-expert sigmoid gate)."""
weight = getattr(module, "weight", None)
return (
isinstance(module, nn.Linear)
and weight is not None
and weight.ndim == 2
and weight.shape[0] == 1
)
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
"""Returns the set of nodes reachable from `node` by following `key` edges."""
seen: set[fx.Node] = set()
stack = [node]
while stack:
n = stack.pop()
if n in seen:
continue
seen.add(n)
stack.extend(getattr(n, key))
return seen
class SharedExpertMLP(nn.Module):
"""Wraps an HF shared expert, applying the output gating it is paired with."""
def __init__(self, shared_experts: nn.Module, gate: nn.Module | None = None):
super().__init__()
self.shared_experts = shared_experts
self.gate = gate
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
out = self.shared_experts(hidden_states)
if self.gate is not None:
out = torch.sigmoid(self.gate(hidden_states)[0]) * out
return out
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
"""Standard MoE block forward.
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
orig_shape = hidden_states.shape
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
num_tokens = hidden_states.shape[0]
is_sequence_parallel = self.experts.moe_config.is_sequence_parallel
if is_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
out = self.experts(hidden_states, router_logits=hidden_states)
if is_sequence_parallel:
out = tensor_model_parallel_all_gather(out, 0)[:num_tokens]
return out.reshape(orig_shape)
@dataclass
class MoEBlockFuser:
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
gate_name: str
scoring_func: str
shared_name: str | None
shared_gate_name: str | None
@staticmethod
def _match_router(gate: nn.Module) -> str | None:
"""Matches `topk(score(linear(x)))`, `score` being `softmax`/`sigmoid`."""
if [name for name, _ in named_state(gate)] != ["weight"]:
return None
graph = trace(gate)
if graph is None:
return None
topk = find_node(graph, lambda n: is_op(n, "topk"))
if topk is None:
return None
# Exactly one scoring op upstream of the top-k, fed (transitively) by a linear.
scorers = [
n
for n in _reaches(topk, "all_input_nodes")
if is_op(n, "softmax") or is_op(n, "sigmoid")
]
if len(scorers) != 1:
return None
scorer = scorers[0]
if not any(is_op(n, "linear") for n in _reaches(scorer, "all_input_nodes")):
return None
return "softmax" if is_op(scorer, "softmax") else "sigmoid"
@staticmethod
def _match_shared_experts(
graph: fx.Graph, experts: str
) -> tuple[str | None, str | None]:
"""Detects the shared expert and its optional gate by dataflow."""
experts_predicate = lambda n: n.op == "call_module" and n.target == experts
if (experts_node := find_node(graph, experts_predicate)) is None:
return None, None
from_experts = _reaches(experts_node, "users")
for add in graph.nodes:
if not is_op(add, "add"):
continue
operands = [a for a in add.args if isinstance(a, fx.Node)]
# Exactly one side is the experts' output; the other is the shared path.
sides = [a in from_experts for a in operands]
if len(operands) != 2 or sides.count(True) != 1:
continue
cone = _reaches(operands[sides.index(False)], "all_input_nodes")
modules = [n for n in cone if n.op == "call_module" and n.target != experts]
# A sigmoid wrapping one of those modules marks the shared-expert gate.
gate = next(
(
src
for n in cone
if is_op(n, "sigmoid")
and isinstance(src := peel(n.args[0]), fx.Node)
and src in modules
),
None,
)
shared = [n for n in modules if n is not gate]
if len(shared) != 1:
return None, None
return shared[0].target, (gate.target if gate is not None else None)
return None, None
@classmethod
def match(cls, moe_block: nn.Module, experts_name: str) -> "MoEBlockFuser | None":
# Standard MoE block returns a single tensor.
if _returns_tuple(type(moe_block)):
return None
# Router: the child that scores + top-k selects.
gate_name = scoring_func = None
for name, child in moe_block.named_children():
if name != experts_name and (func := cls._match_router(child)) is not None:
gate_name, scoring_func = name, func
break
if gate_name is None or scoring_func is None:
return None
# Shared expert: a child the block adds to the experts' output.
shared_name = shared_gate_name = None
others = [
n
for n, _ in moe_block.named_children()
if n not in {experts_name, gate_name}
]
if others:
graph = trace(moe_block)
if graph is None:
return None
shared_name, shared_gate_name = cls._match_shared_experts(
graph, experts_name
)
if shared_gate_name is not None and not _is_scalar_gate(
getattr(moe_block, shared_gate_name)
):
return None
# Fail closed: `rewrite_forward` runs only the experts and the detected
# shared expert, so any other stateful child would be dropped.
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
return cls(gate_name, scoring_func, shared_name, shared_gate_name)
def gate(self, moe_block: nn.Module, prefix: str) -> ReplicatedLinear:
"""Rebuild the HF gate as a `ReplicatedLinear` for vLLM's fused MoE."""
num_experts, hidden_size = getattr(moe_block, self.gate_name).weight.shape
gate = ReplicatedLinear(
hidden_size,
num_experts,
bias=False,
prefix=maybe_prefix(prefix, self.gate_name),
)
setattr(moe_block, self.gate_name, gate)
return gate
def shared_experts(
self, moe_block: nn.Module, prefix: str
) -> SharedExpertMLP | None:
"""Build the HF shared expert (and its optional gate)
as a `SharedExpertMLP` for vLLM's fused MoE."""
if self.shared_name is None:
return None
shared_experts = getattr(moe_block, self.shared_name)
gate = None
if self.shared_gate_name is not None:
hf_gate = getattr(moe_block, self.shared_gate_name)
gate = ReplicatedLinear(
hf_gate.in_features,
hf_gate.out_features,
bias=hf_gate.bias is not None,
prefix=maybe_prefix(prefix, self.shared_gate_name),
)
setattr(moe_block, self.shared_gate_name, gate)
return SharedExpertMLP(shared_experts, gate)
def rewrite_forward(self, moe_block: nn.Module) -> None:
"""Rewrite `moe_block.forward` to route through vLLM's fused MoE."""
moe_block.forward = types.MethodType(_moe_block_forward, moe_block)
@@ -0,0 +1,220 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""QKV projection fuser: `q(x), k(x), v(x)` -> a fused qkv linear + split."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
innermost_block,
is_linear,
recover_forward,
replace_expr,
single_self_call,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
@dataclass
class QKVFuser(StackedFuser):
"""Fuser for the attention QKV pattern `q(x), k(x), v(x)`."""
q_name: str
k_name: str
v_name: str
o_name: str | None
merged_name: ClassVar[str] = "qkv_proj"
merged_cls: ClassVar[str] = "QKVParallelLinear"
@property
def shards(self) -> list[tuple[str, ShardId]]:
return [(self.q_name, "q"), (self.k_name, "k"), (self.v_name, "v")]
@classmethod
def _get_qkv_nodes(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
"""Search `graph` for the QKV pattern `q(x), k(x), v(x)`."""
by_input: dict[fx.Node, list[fx.Node]] = {}
for node in graph.nodes:
if (
is_linear(node, module)
and len(node.args) == 1
and not node.kwargs
and isinstance(node.args[0], fx.Node)
and node.args[0].op == "placeholder"
):
by_input.setdefault(node.args[0], []).append(node)
triples = [nodes for nodes in by_input.values() if len(nodes) == 3]
if len(triples) != 1:
return None
q_node, k_node, v_node = nodes = triples[0]
outs = [module.get_submodule(node.target).out_features for node in nodes]
if len(set(outs)) == 2:
# q is identified as the larger projection (GQA)
(q_node,) = (n for n, out in zip(nodes, outs) if outs.count(out) == 1)
k_node, v_node = (n for n, out in zip(nodes, outs) if outs.count(out) == 2)
if module.get_submodule(q_node.target).out_features != max(outs):
return None
elif len(set(outs)) != 1:
return None
return q_node, k_node, v_node
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "QKVFuser | None":
if (qkv_nodes := cls._get_qkv_nodes(graph, module)) is None:
return None
q, k, v = qkv_nodes
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
attn_width = module.get_submodule(q.target).out_features
candidates = [
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
and name not in names.values()
and child.in_features == attn_width
]
names["o_name"] = candidates[0] if len(candidates) == 1 else None
return cls(source_cls=type(module).__name__, **names)
def update_forward(self, module: nn.Module) -> None:
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source."""
funcdef, fn = recover_forward(type(module))
calls = [
single_self_call(funcdef, name)
for name in (self.q_name, self.k_name, self.v_name)
]
arg_dumps = {ast.dump(call.args[0]) for call in calls}
if len(arg_dumps) != 1:
raise ValueError("projection inputs are written differently")
# The trace may be partial, so prove projection exclusivity in source:
# no other linear child may consume the same input (else the matched
# three may not be q, k and v)
other_linears = {
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
} - {self.q_name, self.k_name, self.v_name}
for node in ast.walk(funcdef):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr in other_linears
and any(ast.dump(arg) in arg_dumps for arg in node.args)
):
raise ValueError("another linear consumes the same input")
blocks = [innermost_block(funcdef.body, call) for call in calls]
if any(found is None for found in blocks):
raise ValueError("projection calls not found in the function body")
if len({id(block) for block, _ in blocks}) != 1:
raise ValueError("projection calls are in different blocks")
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(self.qkv.split_sizes, -1)
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
temps = [f"{name}_fused" for name in (self.q_name, self.k_name, self.v_name)]
if names & set(temps):
raise ValueError("fused temporaries would shadow existing names")
merged = f"self.{self.merged_name}"
template = (
f"{', '.join(temps)} = {merged}(__arg__).split({merged}.split_sizes, -1)"
)
assign = ast.parse(template).body[0]
arg = next(
node
for node in ast.walk(assign)
if isinstance(node, ast.Name) and node.id == "__arg__"
)
replace_expr(assign, arg, calls[0].args[0])
block, index = blocks[0]
ast.copy_location(assign, block[index])
block.insert(min(index for _, index in blocks), assign)
for call, temp in zip(calls, temps):
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
v = module.get_submodule(self.v_name)
head_size = model_config.get_head_size()
compatible = (
q.in_features == k.in_features == v.in_features
and len({proj.bias is None for proj in (q, k, v)}) == 1
and k.out_features == v.out_features
and q.out_features % head_size == 0
and k.out_features % head_size == 0
)
if not compatible:
logger.debug("%s is not compatible with QKV fusion", type(module))
return compatible
def update_attrs(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
) -> None:
head_size = model_config.get_head_size()
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
merged = QKVParallelLinear(
hidden_size=q.in_features,
head_size=head_size,
total_num_heads=q.out_features // head_size,
total_num_kv_heads=k.out_features // head_size,
bias=q.bias is not None,
quant_config=quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
)
logger.debug(
"%s: %s, %s: %s, %s: %s -> %s: %s",
self.q_name,
q,
self.k_name,
k,
self.v_name,
module.get_submodule(self.v_name),
self.merged_name,
merged,
)
# The rewritten forward splits the merged projection into this rank's
# shard sizes (see `update_forward`)
merged.split_sizes = [
merged.num_heads * merged.head_size,
merged.num_kv_heads * merged.head_size,
merged.num_kv_heads * merged.v_head_size,
]
setattr(module, self.merged_name, merged)
# Drop the consumed submodules so their (meta) params are not expected.
for name in (self.q_name, self.k_name, self.v_name):
delattr(module, name)
# If there is an output projection, we know it must be rowwise.
if self.o_name is not None:
o_proj_prefix = maybe_prefix(prefix, self.o_name)
o_proj = module.get_submodule(self.o_name)
new_o = replace_linear_class(
o_proj, "rowwise", quant_config, prefix=o_proj_prefix
)
setattr(module, self.o_name, new_o)
log_replacement(o_proj_prefix, o_proj, new_o)
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RMSNorm fuser: detect the norm structurally and swap in vLLM's fused RMSNorm."""
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
from torch import fx, nn
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
)
from vllm.distributed.parallel_state import model_parallel_is_initialized
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
forward_input_count,
is_op,
peel,
trace,
)
if TYPE_CHECKING:
from vllm.config.model import ModelConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
def _is_squared(node: object, x: fx.Node) -> bool:
"""`x**2`, `x.square()` or `x * x`, through any dtype casts."""
node = peel(node)
if is_op(node, "pow"):
base, exp = node.args
return peel(base) is x and exp == 2
if is_op(node, "square"):
return peel(node.args[0]) is x
if is_op(node, "mul"):
a, b = node.args
return peel(a) is x and peel(b) is x
return False
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
"""eps from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
add = peel(rsqrt.args[0])
if not is_op(add, "add"):
return None
consts = [a for a in add.args if isinstance(a, (int, float))]
nodes = [a for a in add.args if isinstance(a, fx.Node)]
if len(consts) != 1 or len(nodes) != 1:
return None
mean = peel(nodes[0])
if not is_op(mean, "mean"):
return None
if not _is_squared(mean.args[0], x):
return None
return float(consts[0])
def _is_one_plus(node: object) -> bool:
"""`1 + weight` in either operand order (marks a zero-centered weight)."""
node = peel(node)
if not is_op(node, "add"):
return False
return any(isinstance(a, (int, float)) and a == 1 for a in node.args)
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
"""Does the forward compute anything after `node` before returning?"""
output = find_node(graph, lambda n: n.op == "output")
if output is None or not output.args:
return False
return peel(output.args[0]) is not node
class TPAwareNormMixin(nn.Module):
"""Mixin for RMSNorms that reconstructs a TP-sharded input before normalizing."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if model_parallel_is_initialized():
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
else:
self.tp_size, self.tp_rank = 1, 0
def forward(
self, x: torch.Tensor, residual: torch.Tensor | None = None
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if self.tp_size > 1 and x.shape[-1] < (full := self.weight.shape[0]):
if x.shape[-1] * self.tp_size != full:
raise ValueError(
f"Cannot gather norm of width {full}: a TP-sharded input of "
f"width {x.shape[-1]} does not tile it evenly across "
f"{self.tp_size} ranks (replicated or uneven sharding)."
)
x = tensor_model_parallel_all_gather(x.contiguous())
x = super().forward(x)
splits = split_tensor_along_last_dim(x, num_partitions=self.tp_size)
return splits[self.tp_rank]
return super().forward(x, residual)
class TPAwareRMSNorm(TPAwareNormMixin, RMSNorm):
"""`RMSNorm` that reconstructs a TP-sharded input before normalizing."""
class TPAwareGemmaRMSNorm(TPAwareNormMixin, GemmaRMSNorm):
"""`GemmaRMSNorm` that reconstructs a TP-sharded input before normalizing."""
@dataclass
class RMSNormFuser(BaseFuser):
"""Fuser for RMSNorm patterns, including Gemma-style zero-centered weights."""
zero_centered: bool
"""Gemma-style `(1 + weight)` scaling (weight initialised at zero)."""
source_cls: str
"""Class name of the norm this was matched from (for logging)."""
def info(self, name: str) -> str:
norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm"
return f"Fused: {name} ({self.source_cls}) -> {norm} (CustomOp)"
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "RMSNormFuser | None":
"""Match a graph to the RMSNorm pattern, returning a fuser if found."""
if forward_input_count(type(module)) != 1:
return None
x = find_node(graph, lambda n: n.op == "placeholder")
if x is None:
return None
# Handle native torch `rms_norm` op.
rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm"))
if rms_norm is not None and rms_norm.args and peel(rms_norm.args[0]) is x:
if _has_trailing_compute(graph, rms_norm):
return None
return cls(zero_centered=False, source_cls=type(module).__name__)
# Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern.
# The rsqrt over the mean-square variance is the spine of the norm.
rsqrt = None
for node in graph.nodes:
if is_op(node, "rsqrt") and _variance_eps(node, x) is not None:
rsqrt = node
break
if rsqrt is None:
return None
# The `x * rsqrt(...)` normalize multiply.
normalize = find_node(
graph, lambda n: is_op(n, "mul") and rsqrt in map(peel, n.args)
)
if normalize is None:
return None
# An optional later `weight * normalized` (or `(1 + weight) * normalized`).
tail, zero_centered = normalize, False
for node in graph.nodes:
if not is_op(node, "mul") or node is normalize:
continue
operands = [peel(a) for a in node.args if isinstance(a, fx.Node)]
if len(operands) == 2 and normalize in operands:
weight = next(o for o in operands if o is not normalize)
tail, zero_centered = node, _is_one_plus(weight)
break
# The norm must be the last compute in forward, or it is not a pure norm.
if _has_trailing_compute(graph, tail):
return None
return cls(zero_centered=zero_centered, source_cls=type(module).__name__)
@staticmethod
def _eps_from_graph(graph: fx.Graph) -> float | None:
"""Extract the `eps` constant from the graph, if present."""
if (x := find_node(graph, lambda n: n.op == "placeholder")) is None:
return None
fused = find_node(graph, lambda n: is_op(n, "rms_norm"))
if fused is not None and fused.args and peel(fused.args[0]) is x:
args, kwargs = fused.args, fused.kwargs
eps = args[3] if len(args) > 3 else kwargs.get("eps")
return eps if isinstance(eps, (int, float)) else None
for node in graph.nodes:
if is_op(node, "rsqrt") and (eps := _variance_eps(node, x)) is not None:
return eps
return None
def validate(self, module: nn.Module, model_config: "ModelConfig") -> bool:
return True
def fuse(
self,
module: nn.Module,
prefix: str,
model_config: "ModelConfig",
quant_config: "QuantizationConfig",
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
weight = getattr(module, "weight", None)
hidden_size = (
weight.size(0) if weight is not None else model_config.get_hidden_size()
)
graph = trace(module)
eps = self._eps_from_graph(graph) if graph is not None else None
if eps is None:
# If eps not in graph, match torch behaviour.
dtype = weight.dtype if weight is not None else model_config.dtype
eps = torch.finfo(dtype).eps
if self.zero_centered:
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
has_weight = weight is not None
return TPAwareRMSNorm(
hidden_size=hidden_size,
eps=eps,
has_weight=has_weight,
dtype=weight.dtype if has_weight else None,
)
@@ -0,0 +1,273 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fx tracing and forward-source rewriting for the Transformers backend fusers.
A small engine, independent of any particular pattern: trace a module's forward
with `torch.fx` (tolerating a partial graph), inspect the resulting nodes, and
rewrite the forward's *source* (AST) so only matched calls change while the rest
stays live Python. `fusion.py` builds the concrete fusion patterns on top.
"""
import ast
import inspect
import operator
import textwrap
from collections.abc import Callable
import torch
from torch import fx, nn
from torch.nn import functional as F
from vllm.logger import init_logger
logger = init_logger(__name__)
def _infer_len(node: fx.Node) -> int | None:
"""Concrete length of a proxy's value, inferred from its node chain.
Lets tracing pass through the shape unpacks and `*`-splats (e.g.
`(*input_shape, -1, head_dim)`) that precede the patterns in HF attention.
"""
# `x.shape` has the rank of `x`, when known
if (
node.op == "call_function"
and node.target is getattr
and node.args[1] == "shape"
and (rank := _rank(node.args[0])) is not None
):
return rank
# Slices of known-length values
if node.op == "call_function" and node.target is operator.getitem:
src_len = _infer_len(node.args[0])
index = node.args[1]
if src_len is not None and isinstance(index, slice):
return len(range(*index.indices(src_len)))
return None
def _rank(node: fx.Node) -> int | None:
"""The tensor rank of `node`'s value, if known."""
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states
if node.op == "placeholder" and node.target == "hidden_states":
return 3
return None
class _SizedProxy(fx.Proxy):
"""Proxy whose `len` is inferred from the graph (see `_infer_len`)."""
def __len__(self) -> int:
length = _infer_len(self.node)
if length is None:
return super().__len__()
return length
class _AllLeafTracer(fx.Tracer):
"""Tracer that treats every submodule as a leaf.
Each child stays one `call_module` node, so matching sees the module's own
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
`iter` traces through the leading shape unpacks (see `_infer_len`); anything
else untraceable ends the trace early and the partial graph is matched.
"""
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
return True
def proxy(self, node: fx.Node) -> fx.Proxy:
return _SizedProxy(node, self)
def iter(self, obj: fx.Proxy):
length = _infer_len(obj.node)
if length is None:
return super().iter(obj)
return iter([obj[i] for i in range(length)])
def trace(module: nn.Module) -> fx.Graph | None:
"""Trace `module.forward`, returning the partial graph on failure.
The graph is only evidence for matching, and the patterns sit at the top of
their forwards, so a trace that fails partway can still be matched."""
tracer = _AllLeafTracer()
try:
return tracer.trace(module)
except Exception as exc:
logger.debug("Could not fully trace %s: %s", type(module), exc)
return getattr(tracer, "graph", None)
def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
"""Parse the source of `cls.forward`, ready for rewriting."""
fn = inspect.unwrap(cls.forward)
if fn.__code__.co_freevars:
raise ValueError("forward is a closure")
tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
funcdef = tree.body[0]
if not isinstance(funcdef, ast.FunctionDef):
raise ValueError("source is not a plain function definition")
# `fn` is already unwrapped; don't re-apply its decorators
funcdef.decorator_list.clear()
# Annotations may not evaluate outside the defining module (e.g. with
# postponed evaluation); they're not needed at runtime
funcdef.returns = None
args = funcdef.args
for arg in (
*args.posonlyargs,
*args.args,
*args.kwonlyargs,
*filter(None, (args.vararg, args.kwarg)),
):
arg.annotation = None
# Recompiling outside the class body would break name mangling
for node in ast.walk(funcdef):
name = getattr(node, "attr", None) or getattr(node, "id", None)
if name and name.startswith("__") and not name.endswith("__"):
raise ValueError(f"{name} would be name mangled")
return funcdef, fn
def forward_input_count(cls: type[nn.Module]) -> int:
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
any `*args`/`**kwargs`. Read from the signature, so it is independent of
whether the trace completes (unlike counting placeholders)."""
try:
params = list(inspect.signature(cls.forward).parameters.values())[1:]
except (ValueError, TypeError):
return 1 # uninspectable: assume a single input and let matching decide
fixed = (
inspect.Parameter.POSITIONAL_ONLY,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
)
return sum(1 for p in params if p.kind in fixed)
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
"""Compile `funcdef` in `fn`'s module so tracebacks point at the source."""
module = ast.Module(body=[funcdef], type_ignores=[])
ast.fix_missing_locations(module)
ast.increment_lineno(module, fn.__code__.co_firstlineno - 1)
code = compile(module, fn.__code__.co_filename, "exec")
namespace: dict = {}
exec(code, fn.__globals__, namespace)
return namespace[funcdef.name]
def single_self_call(funcdef: ast.FunctionDef, name: str) -> ast.Call:
"""The unique `self.<name>(arg)` call in `funcdef`.
Raises unless `name` appears exactly once, as such a call, so the source
rewrite agrees with the fx match.
"""
uses = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Attribute) and node.attr == name
]
if len(uses) != 1:
raise ValueError(f"{name} is referenced {len(uses)} times")
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and node.func is uses[0]
and len(node.args) == 1
and not isinstance(node.args[0], ast.Starred)
and not node.keywords
]
if (
len(calls) != 1
or not isinstance(uses[0].value, ast.Name)
or uses[0].value.id != "self"
):
raise ValueError(f"{name} is not a single-argument call on self")
return calls[0]
def innermost_block(
block: list[ast.stmt], node: ast.AST
) -> tuple[list[ast.stmt], int] | None:
"""The innermost statement list containing `node`, and the index within."""
for index, stmt in enumerate(block):
if not any(child is node for child in ast.walk(stmt)):
continue
child_blocks = [
getattr(stmt, fld, None) for fld in ("body", "orelse", "finalbody")
]
child_blocks += [h.body for h in getattr(stmt, "handlers", [])]
child_blocks += [c.body for c in getattr(stmt, "cases", [])]
for child_block in child_blocks:
if (
isinstance(child_block, list)
and child_block
and (found := innermost_block(child_block, node)) is not None
):
return found
return block, index
return None
def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
"""Replace the expression `old` (by identity) with `new` within `module`."""
class _Replacer(ast.NodeTransformer):
def visit(self, node: ast.AST) -> ast.AST:
if node is old:
return new
return super().generic_visit(node)
_Replacer().visit(module)
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
"""The first node in `graph` matching `predicate`, or `None`."""
return next((n for n in graph.nodes if predicate(n)), None)
def is_linear(node: fx.Node, module: nn.Module) -> bool:
"""Is node `nn.Linear.__call__()`."""
return node.op == "call_module" and isinstance(
module.get_submodule(node.target), nn.Linear
)
_DTYPE_CASTS = frozenset({"to", "float", "double", "half", "bfloat16", "type_as"})
def peel(node: object) -> object:
"""Strip dtype-cast wrappers (`.to(...)`, `.float()`, `.type_as(...)`)."""
while (
isinstance(node, fx.Node)
and node.op == "call_method"
and node.target in _DTYPE_CASTS
):
node = node.args[0]
return node
def is_fn(node: object, target: Callable) -> bool:
"""Is node `<target>()`."""
return (
isinstance(node, fx.Node)
and node.op == "call_function"
and node.target is target
)
def is_method(node: object, name: str) -> bool:
"""Is node `.<name>()`."""
return (
isinstance(node, fx.Node) and node.op == "call_method" and node.target == name
)
def is_op(node: object, name: str) -> bool:
"""
Is node `torch.<name>()`, `F.<name>()`, `operator.<name>()`, or `Tensor.<name>()`.
"""
return any(
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
) or (hasattr(torch.Tensor, name) and is_method(node, name))
+83 -46
View File
@@ -26,9 +26,11 @@ import torch.nn as nn
from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import FusedMoE, MoERunner, RoutedExperts
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import maybe_prefix
from vllm.utils.torch_utils import direct_register_custom_op
@@ -37,6 +39,8 @@ from .utils import log_replacement
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class TransformersMoEState:
@@ -142,14 +146,15 @@ class MoEMixin(MixtureOfExperts):
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for mlp in self.mlp_layers:
mlp.n_local_physical_experts = num_local_physical_experts
mlp.n_physical_experts = num_physical_experts
mlp.n_redundant_experts = self.num_redundant_experts
mlp.experts.update_expert_map()
for moe_block in self.mlp_layers:
moe_block.n_local_physical_experts = num_local_physical_experts
moe_block.n_physical_experts = num_physical_experts
moe_block.n_redundant_experts = self.num_redundant_experts
moe_block.experts.update_expert_map()
def recursive_replace(self):
"""Initialize the MoE layers."""
experts_name = "experts"
text_config = self.text_config
# Positional arguments
@@ -217,10 +222,13 @@ class MoEMixin(MixtureOfExperts):
# down_proj = (num_experts, intermediate_size, hidden_size)
params = list(child_module.parameters())
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
if child_name == "experts" and (is_modulelist or is_3d):
if child_name == experts_name and (is_modulelist or is_3d):
# Alias for readability
mlp = module
moe_block = module
experts = child_module
# Class of the fused block (parent of gate/experts/shared)
moe_block_cls = type(moe_block).__name__
experts_cls = type(experts).__name__
# Do the experts have biases
has_bias = False
for experts_param_name, _ in experts.named_parameters():
@@ -230,64 +238,93 @@ class MoEMixin(MixtureOfExperts):
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
for mlp_param_name, _ in mlp.named_parameters():
if "shared_expert" in mlp_param_name:
for moe_block_param_name, _ in moe_block.named_parameters():
if "shared_expert" in moe_block_param_name:
self.num_shared_experts = 1
break
# Replace experts module with FusedMoE
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
dist_group = get_ep_group() if is_sp else get_dp_group()
assert sizes[dist_group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = dist_group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
fused_experts = FusedMoE(
kwargs: dict[str, Any] = dict(
num_experts=num_experts,
top_k=top_k,
hidden_size=hidden_size,
intermediate_size=intermediate_size,
renormalize=renormalize,
# Hard coded because topk happens in Transformers
use_grouped_topk=False,
num_expert_group=num_expert_group,
topk_group=topk_group,
quant_config=self.quant_config,
prefix=qual_name,
activation=activation,
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
custom_routing_function=partial(
custom_routing_function,
moe_state=moe_state,
),
runner_cls=TransformersMoERunner,
routed_experts_cls=TransformersRoutedExperts,
runner_args={"moe_state": moe_state},
)
mlp.experts = fused_experts
fuser = MoEBlockFuser.match(moe_block, experts_name)
if self.num_expert_groups <= 1 and fuser is not None:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in FusedMoE.
kwargs |= dict(
scoring_func=fuser.scoring_func,
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
gate=fuser.gate(moe_block, prefix),
shared_experts=fuser.shared_experts(moe_block, prefix),
)
fuser.rewrite_forward(moe_block)
routed = "gate + experts"
if fuser.shared_name:
routed += " + shared experts"
logger.info_once(
"Fused: %s (%s) -> FusedMoE (internal routing)",
routed,
moe_block_cls,
)
else:
# MoE block forward is unmodified.
# gate/router and shared expert (if any) runs in Transformers.
# We then smuggle the topk_ids in using a custom op.
moe_state = TransformersMoEState()
def custom_routing_function(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
):
"""Return `topk_weights` from `gating_output` and the
`topk_ids` we stored in the layer earlier."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
custom_routing_function, moe_state=moe_state
),
runner_cls=TransformersMoERunner,
runner_args={"moe_state": moe_state},
)
logger.info_once(
"Fused: experts (%s) -> FusedMoE (external routing)",
experts_cls,
)
fused_experts = FusedMoE(**kwargs)
moe_block.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
self.mlp_layers.append(mlp)
self.mlp_layers.append(moe_block)
self.moe_layers.append(fused_experts)
else:
_recursive_replace(child_module, prefix=qual_name)
@@ -23,10 +23,8 @@ from typing import TYPE_CHECKING, Literal
import torch
from torch import nn
from vllm.config.utils import getattr_iter
from vllm.logger import init_logger
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.layernorm import GemmaRMSNorm, RMSNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
@@ -183,45 +181,6 @@ def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
)
def replace_rms_norm_class(rms_norm: nn.Module, hidden_size: int) -> RMSNorm:
"""Replace a Transformers RMSNorm with vLLM's RMSNorm.
This method assumes:
- Weight is stored as `weight`.
- Epsilon is stored as `eps` or `variance_epsilon`.
- `with_scale` indicates whether the layer has a weight (Gemma3n only).
- `var_hidden_size` is only ever used for Intern vision encoder in vLLM
and Transformers doesn't appear to have the same concept.
"""
eps = getattr_iter(rms_norm, ("eps", "variance_epsilon"), 1e-6)
kwargs = {"hidden_size": hidden_size, "eps": eps}
# Update hidden size if weight is available
weight_meta = getattr(rms_norm, "weight", None)
if weight_meta is not None:
kwargs["hidden_size"] = weight_meta.size(0)
# Check if weight is all zeros, which indicates GemmaRMSNorm
# We must create a new instance because rms_norm is on meta
try:
with torch.device("cpu"):
weight_test = getattr(rms_norm.__class__(1), "weight", None)
except Exception:
logger.warning(
"Failed to determine if RMSNorm weight is centered on zero or one. "
"Defaulting to one."
)
weight_test = None
if weight_test is not None and torch.all(weight_test == 0):
return GemmaRMSNorm(**kwargs)
# Otherwise assume it's a regular RMSNorm
kwargs["has_weight"] = getattr(rms_norm, "with_scale", True)
if weight_meta is not None:
kwargs["dtype"] = weight_meta.dtype
else:
# No weight, fall back to weightless RMSNorm
kwargs["has_weight"] = False
return RMSNorm(**kwargs)
def recursive_replace_linear(
model: nn.Module,
quant_config: "QuantizationConfig | None",