[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers

- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs,
  moriio, hisparse, example and hidden-states connectors
- Reduce weight_transfer to the base API used by model loading
- Remove kv_offload tiering, simple_kv_offload, ec cpu/example,
  snapshot command, engine snapshot server
- Remove transformers model fallback, dsv4 tokenizers, dead platforms
  cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn
  ops, dead layers (short_conv, fused_embed_norm, indexer_topk,
  relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar
- Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant
- Rebuild parser hubs around Qwen3 adapters only (36 dead adapters)
- Remove dead tests with the code they covered

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 02:19:03 +07:00
co-authored by Hinata
parent 54a396586e
commit 2b544b7ba0
353 changed files with 29 additions and 121269 deletions
+2 -70
View File
@@ -115,77 +115,9 @@ _SPECULATIVE_DECODING_MODELS = {
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
}
_TRANSFORMERS_SUPPORTED_MODELS = {
# Text generation models
"ArceeForCausalLM": ("transformers", "TransformersForCausalLM"),
"CwmForCausalLM": ("transformers", "TransformersForCausalLM"),
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
"GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"),
"PhiForCausalLM": ("transformers", "TransformersForCausalLM"),
"SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"),
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
# Multimodal models
"Emu3ForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
"HunYuanVLForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
"VibeVoiceAsrForConditionalGeneration": (
"transformers",
"TransformersMultiModalForCausalLM",
),
}
_TRANSFORMERS_SUPPORTED_MODELS = {}
_TRANSFORMERS_BACKEND_MODELS = {
# Text generation models
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
"TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
# Multimodal models
"TransformersMultiModalForCausalLM": (
"transformers",
"TransformersMultiModalForCausalLM",
),
"TransformersMultiModalMoEForCausalLM": (
"transformers",
"TransformersMultiModalMoEForCausalLM",
),
# Embedding models
"TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
"TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
"TransformersMultiModalEmbeddingModel": (
"transformers",
"TransformersMultiModalEmbeddingModel",
),
# Sequence classification models
"TransformersForSequenceClassification": (
"transformers",
"TransformersForSequenceClassification",
),
"TransformersMoEForSequenceClassification": (
"transformers",
"TransformersMoEForSequenceClassification",
),
"TransformersMultiModalForSequenceClassification": (
"transformers",
"TransformersMultiModalForSequenceClassification",
),
}
_TRANSFORMERS_BACKEND_MODELS = {}
_VLLM_MODELS = {
**_TEXT_GENERATION_MODELS,
@@ -1,155 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Wrapper around `transformers` models."""
from typing import TYPE_CHECKING
import torch.nn.functional as F
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR
from vllm.model_executor.models.transformers.causal import CausalMixin
from vllm.model_executor.models.transformers.moe import MoEMixin
from vllm.model_executor.models.transformers.multimodal import (
MultiModalDummyInputsBuilder,
MultiModalMixin,
MultiModalProcessingInfo,
MultiModalProcessor,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
if TYPE_CHECKING:
import torch
from vllm.model_executor.layers.attention import Attention, MLAAttention
def check_sinks(
module: "torch.nn.Module",
self_attn: "Attention | MLAAttention",
s_aux: "torch.Tensor | None",
):
"""Fail loudly if the model applies a sink the attention layer will not.
Only the attention impl can fold a sink into the softmax denominator, so a sink
that never reached `Attention` is dropped and every softmax is subtly wrong.
"""
if s_aux is None or getattr(self_attn, "has_sink", False):
return
raise ValueError(
f"{type(module).__name__} applies attention sinks, but they were not passed "
f"to {type(self_attn).__name__}, so the output would be wrong. Either the "
"Transformers modeling backend could not find the parameter holding them, or "
"vLLM does not support sinks for this kind of attention. Please open an issue "
"at https://github.com/vllm-project/vllm/issues/new"
)
def vllm_attention_forward(
# Transformers args
module: "torch.nn.Module",
query: "torch.Tensor",
key: "torch.Tensor",
value: "torch.Tensor",
attention_mask: "torch.Tensor",
**kwargs,
):
self_attn = getattr(module, VLLM_ATTN_ATTR)
check_sinks(module, self_attn, kwargs.get("s_aux"))
hidden = query.shape[-2]
head_dim_qk = query.shape[-1]
head_dim_v = value.shape[-1]
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
# Pad `value` up to the query/key head size when it is smaller (expanded
# MLA). A larger last dim just means `value` isn't split per head, e.g.
# packed grouped/multi-query projections, and needs no padding.
pad_value = head_dim_v < head_dim_qk
if pad_value:
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
value = value.reshape(hidden, -1)
attn_output = self_attn.forward(query, key, value)
if pad_value:
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
attn_output = attn_output.reshape(hidden, -1)
return attn_output, None
def vllm_mla_attention_forward(
# Transformers args
module: "torch.nn.Module",
query: "torch.Tensor",
kv_c_normed: "torch.Tensor",
k_pe: "torch.Tensor",
attention_mask: "torch.Tensor",
**kwargs,
):
self_attn = getattr(module, VLLM_ATTN_ATTR)
check_sinks(module, self_attn, kwargs.get("s_aux"))
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
query = query.transpose(1, 2).flatten(0, 1)
num_tokens, num_heads = query.shape[:2]
# [batch=1, num_tokens, kv_lora_rank] -> [num_tokens, kv_lora_rank]
kv_c_normed = kv_c_normed.reshape(-1, kv_c_normed.shape[-1])
# [batch=1, heads=1, num_tokens, qk_rope] -> [num_tokens, 1, qk_rope]
k_pe = k_pe.reshape(-1, 1, k_pe.shape[-1])
attn_output = self_attn.forward(
query,
kv_c_normed,
k_pe,
output_shape=(num_tokens, num_heads * self_attn.v_head_dim),
)
return attn_output, None
ALL_ATTENTION_FUNCTIONS.register("vllm", vllm_attention_forward)
ALL_ATTENTION_FUNCTIONS.register("vllm_mla", vllm_mla_attention_forward)
# Text only models
class TransformersForCausalLM(CausalMixin): ...
class TransformersMoEForCausalLM(MoEMixin, CausalMixin): ...
# Multimodal models
@MULTIMODAL_REGISTRY.register_processor(
MultiModalProcessor,
info=MultiModalProcessingInfo,
dummy_inputs=MultiModalDummyInputsBuilder,
)
class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ...
@MULTIMODAL_REGISTRY.register_processor(
MultiModalProcessor,
info=MultiModalProcessingInfo,
dummy_inputs=MultiModalDummyInputsBuilder,
)
class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ...
def __getattr__(name: str):
"""Handle imports of non-existent classes with a helpful error message."""
if name not in globals():
raise AttributeError(
"The Transformers modeling backend does not currently have a class to "
f"handle the requested model type: {name}. Please open an issue at "
"https://github.com/vllm-project/vllm/issues/new"
)
return globals()[name]
@@ -1,860 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend base class."""
import os
from collections.abc import Callable, Iterable
from contextlib import contextmanager
from functools import cached_property
from itertools import chain
from operator import attrgetter
from typing import TYPE_CHECKING, Any, NamedTuple
import regex as re
import torch
import transformers
from packaging.version import Version
from torch import nn
from transformers import AutoModel, PreTrainedConfig
from transformers.conversion_mapping import (
WeightRenaming,
get_checkpoint_conversion_mapping,
get_model_conversion_mapping,
)
from vllm.compilation.decorators import DynamicArgDims, support_torch_compile
from vllm.config.utils import getattr_iter
from vllm.distributed import get_pp_group, get_tp_group
from vllm.distributed.utils import get_pp_indices
from vllm.logger import init_logger
from vllm.model_executor.layers.attention import Attention, EncoderOnlyAttention
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
from vllm.model_executor.layers.fused_moe import MoERunner
from vllm.model_executor.models.interfaces import (
SupportsEagle,
SupportsEagle3,
SupportsLoRA,
SupportsPP,
SupportsQuant,
)
from vllm.model_executor.models.interfaces_base import VllmModel
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
from vllm.model_executor.models.transformers.fusers.attention import (
VLLM_ATTN_IMPL,
VLLM_MLA_ATTN_IMPL,
)
from vllm.model_executor.models.transformers.utils import (
attrsetter,
can_enable_torch_compile,
disable_noop_dynamic_rope,
get_feature_request_tip,
init_on_device_without_buffers,
log_replacement,
named_state,
replace_conv_class,
replace_embedding_class,
replace_layernorm_class,
replace_linear_class,
)
from vllm.model_executor.models.utils import (
AutoWeightsLoader,
PPMissingLayer,
ShardId,
WeightsMapper,
make_empty_intermediate_tensors_factory,
maybe_prefix,
)
from vllm.sequence import IntermediateTensors
if TYPE_CHECKING:
from transformers import PreTrainedModel
from vllm.config import VllmConfig
try:
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
except ImportError: # MLA support removed from this build
MLAAttention = type("AbsentMLAAttention", (), {})
logger = init_logger(__name__)
VLLM_ATTN_ATTR = "attn"
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
attention module. `"attn"` is chosen deliberately to match the mappings generated by
`maybe_remap_kv_scale_name`."""
class PreTrainedModelClasses(NamedTuple):
decoder: type["PreTrainedModel"]
encoders: dict[str, type["PreTrainedModel"]]
"""Modality -> encoder class, for each modality that has one."""
class Base(
nn.Module,
VllmModel,
SupportsQuant,
SupportsLoRA,
SupportsPP,
SupportsEagle,
SupportsEagle3,
):
hf_to_vllm_mapper: WeightsMapper
# TODO transformers will have a util to get these
embedding_modules = {"embed_tokens": "input_embeddings"}
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
nn.Module.__init__(self)
logger.info("Using Transformers modeling backend.")
self.vllm_config = vllm_config
self.config = vllm_config.model_config.hf_config
self.text_config = self.config.get_text_config()
self.cache_config = vllm_config.cache_config
self.compilation_config = vllm_config.compilation_config
self.device_config = vllm_config.device_config
self.model_config = vllm_config.model_config
self.parallel_config = vllm_config.parallel_config
self.quant_config = vllm_config.quant_config
self.pp_group = get_pp_group()
self.tp_group = get_tp_group()
# Attrs for weight loading (see self.load_weights)
self.ignore_unexpected_prefixes: list[str] = []
"""Ignore unexpected weights whose qualname starts with these prefixes."""
self.ignore_unexpected_suffixes: list[str] = []
"""Ignore unexpected weights whose qualname ends with these suffixes."""
self.packed_modules_mapping: dict[str, list[str]] = {}
"""Fused module -> constituent projections, populated by `recursive_replace`
for the quantization machinery and loaders (e.g. bitsandbytes)."""
self.fusers: dict[str, list[BaseFuser]] = {}
"""Module qualname -> the fusers applied to it, populated
by `recursive_replace` for `_create_attention_instances`."""
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
"""`layer_idx` -> the qualname and fuser of the module computing that
layer's attention, populated by `recursive_replace` for
`_create_attention_instances`."""
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
self._target_class: type[nn.Module] = nn.Module
"""Target class for Eagle3 aux hidden state recording."""
self._layer_names: dict[int, str] = {}
"""Mapping from layer index to layer name for Eagle3."""
self._output_aux_hidden_states_kwargs: dict[str, bool] = {}
"""Kwargs to pass to model forward for Eagle3 aux hidden states."""
if self.quant_config:
quant_method_name = self.quant_config.get_name()
# Check for unsupported quantization methods.
if quant_method_name in ("mxfp4", "gpt_oss_mxfp4"):
raise NotImplementedError(
"Transformers modeling backend does "
"not support MXFP4 quantization yet."
)
self._patch_config()
self._decorate_for_torch_compile()
# Init on "meta" to delay allocating GPU tensors
with (
self._mark_model_components(vllm_config),
init_on_device_without_buffers("meta"),
):
from_config_kwargs = self._from_config_kwargs
self.model: PreTrainedModel = AutoModel.from_config(**from_config_kwargs)
# Create weight name to module qualname mapper
self._create_hf_to_vllm_mapper()
# Remove layers not on this pipeline parallel rank
self.pipeline_parallel()
# Skip the device sync of dynamic rope where it can never rescale
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
# Substitute remaining layers with vLLM's layers as needed
self.recursive_replace()
# Create attention instances for KV cache allocation
self._create_attention_instances(prefix)
# Initialize any parameters that have not had their modules replaced
self.init_parameters(self.model)
# Upcast weights Transformers always keeps in fp32
self.keep_in_fp32(self.model)
# Pipeline parallel intermediate tensors
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states"], self.text_config.hidden_size
)
def _patch_config(self):
"""Patch the config to ensure that the model is created correctly:
- Sets the attention implementation to "vllm" so the attention instances from
`_create_attention_instances` are used
- Sets the dtype to the default torch dtype set by vLLM because Transformers
uses the config dtype when creating the model
"""
self.text_config._attn_implementation = VLLM_ATTN_IMPL
self.config.dtype = torch.get_default_dtype()
@contextmanager
def _mark_model_components(self, vllm_config: "VllmConfig"):
"""Mark language model and tower submodules as `self.model` is created.
Nothing to do in `Base`, `MultiModalMixin` will override."""
yield
@cached_property
def _from_config_kwargs(self) -> dict[str, Any]:
"""The kwargs used to create `self.model`."""
return dict(
config=self.config,
dtype=self.model_config.dtype,
trust_remote_code=self.model_config.trust_remote_code,
)
def _find_encoder_classes(
self, model: "PreTrainedModel"
) -> dict[str, type["PreTrainedModel"]]:
"""Find the encoder class of each modality `model` has one for.
Text models have none. Multi-modal ones override this.
"""
return {}
@cached_property
def _pre_trained_model_classes(self) -> PreTrainedModelClasses:
"""The decoder class and the encoder class of each modality that has one.
Both come from a single throwaway model, since building one is not cheap.
"""
with torch.device("meta"):
model: PreTrainedModel = AutoModel.from_config(**self._from_config_kwargs)
model_classes = PreTrainedModelClasses(
decoder=type(model.get_decoder()),
encoders=self._find_encoder_classes(model),
)
del model
logger.debug("Identified model classes as: %s", model_classes)
return model_classes
def _decorate_cls_for_torch_compile(
self,
cls: type["PreTrainedModel"],
dynamic_arg_dims: DynamicArgDims | None,
enable_if: Callable[["VllmConfig"], bool],
is_encoder: bool,
):
"""Decorate `cls` to indicate to vLLM that it supports torch compile.
Args:
cls: The PreTrainedModel class to decorate.
dynamic_arg_dims: A mapping from argument name to the dynamic dimensions
of the argument. If None, default dynamic arg dims will be used. See
[`support_torch_compile`][vllm.compilation.decorators.support_torch_compile]
for more details.
enable_if: A function which takes in the vLLM config and returns whether
torch compile should be enabled for this class.
is_encoder: Whether the class being decorated is an encoder.
"""
logger.debug(
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
cls.__name__,
"encoder" if is_encoder else "decoder",
dynamic_arg_dims,
)
support_torch_compile(
dynamic_arg_dims=dynamic_arg_dims,
enable_if=enable_if,
is_encoder=is_encoder,
use_aot_compile=False,
)(cls)
def _decorate_for_torch_compile(self):
"""Decorate the model's decoder class to indicate to vLLM that it
supports torch compile if `can_enable_torch_compile` is True."""
self._decorate_cls_for_torch_compile(
cls=self._pre_trained_model_classes.decoder,
# Applied to a PreTrainedModel so the batch dimension will exist
dynamic_arg_dims=DynamicArgDims(
input_ids=1, # shape: [1, seq_len]
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
),
enable_if=can_enable_torch_compile,
is_encoder=False,
)
def _create_hf_to_vllm_mapper(self):
"""Create a WeightsMapper to map checkpoint weight names to module qualnames.
This handles:
- Transformers weight renaming from `WeightRenaming`
- Checkpoints saved with a base model prefix that is not `model`
- Checkpoints saved with no base model prefix
- Any quantization config specific mappings
"""
orig_to_new_renaming: list[WeightRenaming] = []
orig_to_new_regex: dict[re.Pattern, str | None] = {}
mappings = get_model_conversion_mapping(self.model)
# Ensure layers that are part of the model, but not AutoModel, get their mapping
instantiated = {type(m).__name__ for m in self.model.modules()}
for arch in self.config.architectures or []:
if arch not in instantiated:
mappings.extend(get_checkpoint_conversion_mapping(arch) or [])
for mapping in mappings:
# Handle weights which have been renamed in Transformers
if isinstance(mapping, WeightRenaming):
orig_to_new_renaming.append(mapping)
# TODO: Handle WeightConverter to enable layer merging
# Handle unexpected weights which should be ignored
if self.model._keys_to_ignore_on_load_unexpected is not None:
for key in self.model._keys_to_ignore_on_load_unexpected:
orig_to_new_regex[re.compile(key)] = None
# Standardise base model prefix
bmp = self.model.base_model_prefix
expected_bmp = r"model.\1"
# Handle checkpoints saved with different base model prefix
if bmp and bmp != "model":
different_bmp_pattern = re.compile(rf"^{bmp}\.(.+)")
orig_to_new_regex[different_bmp_pattern] = expected_bmp
# Handle direct children of self.model which were saved without the model prefix
direct_children = chain(
self.model.named_children(),
self.model.named_parameters(recurse=False),
self.model.named_buffers(recurse=False),
)
model_children = "|".join(name for name, _ in direct_children)
missing_bmp_pattern = re.compile(rf"^(?!model\.)(({model_children}).*)")
orig_to_new_regex[missing_bmp_pattern] = expected_bmp
# Handle weights saved as direct children of self.model which no longer are
unexpected_bmp_pattern = re.compile(rf"^(model\.)((?!{model_children}).+)")
orig_to_new_regex[unexpected_bmp_pattern] = r"\2"
# Handle lm_head which was saved inside the base model
nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)")
orig_to_new_regex[nested_lm_head_pattern] = r"\2"
self.hf_to_vllm_mapper = WeightsMapper(
orig_to_new_renaming=orig_to_new_renaming,
orig_to_new_regex=orig_to_new_regex,
)
# Apply mapping to quantization config if needed
self._maybe_apply_model_mapping()
def _get_tie_word_embeddings(self):
"""Check if the model has tied word embeddings."""
# Models created with Transformers v4 and v5 will store this in different places
tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False)
tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False)
return tie_word_embeddings_v4 or tie_word_embeddings_v5
def pipeline_parallel(self):
"""Apply the model's pipeline parallelization plan."""
if self.pp_group.world_size <= 1:
return
if self.model.supports_pp_plan:
module = self.model
names = list(module._pp_plan.keys())
else:
module = self.model.get_decoder()
has_parameters = lambda m: next(m.parameters(), None) is not None
names = [n for n, c in module.named_children() if has_parameters(c)]
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
logger.warning_once(
"%s does not define a pipeline parallel plan. The Transformers "
"modeling backend will infer the split from the layers of %s in order "
"of declaration and keep parameter-free modules on every rank. This "
"may fail if the model's structure is non-standard. %s",
type(self.model).__name__,
type(module).__name__,
tip,
)
module_lists = []
module_list_idx = None
for i, name in enumerate(names):
# attrgetter in case the module is nested (e.g. "text_model.layers")
if isinstance(attrgetter(name)(module), nn.ModuleList):
module_lists.append(name)
module_list_idx = i
if len(module_lists) > 1:
raise ValueError(
"Pipeline parallel of models with multiple `ModuleList`s "
"in the base model are not supported yet!"
)
if module_list_idx is None:
raise ValueError(f"Could not find `ModuleList` in {type(module)}")
# Layers before module list
for name in names[:module_list_idx]:
if self.pp_group.is_first_rank or (
self._get_tie_word_embeddings() and self.pp_group.is_last_rank
):
continue
# attrsetter in case the module is nested (e.g. "text_model.embed_tokens")
attrsetter(name)(module, PPMissingLayer())
# Module list
start_layer, end_layer = get_pp_indices(
self.text_config.num_hidden_layers,
self.pp_group.rank_in_group,
self.pp_group.world_size,
)
layers_name = names[module_list_idx]
# attrgetter in case the module is nested (e.g. "text_model.layers")
layers = attrgetter(layers_name)(module)
for i in range(len(layers)):
if start_layer <= i and i < end_layer:
continue
layers[i] = PPMissingLayer()
# Layers after module list
for name in names[module_list_idx + 1 :]:
# Modules that should be on last rank
if not self.pp_group.is_last_rank:
# attrsetter in case the module is nested (e.g. "text_model.norm")
attrsetter(name)(module, PPMissingLayer())
def _vocab_embeddings(self) -> set[nn.Embedding]:
"""The `nn.Embedding`s in `self.model` which hold a vocab table."""
def vocab_sizes(config: PreTrainedConfig):
for key, value in vars(config).items():
if isinstance(value, PreTrainedConfig):
yield from vocab_sizes(value)
elif "vocab_size" in key and isinstance(value, int):
yield value
sizes = set(vocab_sizes(self.config))
# `get_input_embeddings` may return a module which composes the `nn.Embedding`,
# or a `PPMissingLayer` if the embeddings are not on this pipeline stage
embeddings = {
module
for module in self.model.get_input_embeddings().modules()
if isinstance(module, nn.Embedding)
}
embeddings.update(
module
for module in self.model.modules()
if isinstance(module, nn.Embedding) and module.num_embeddings in sizes
)
return embeddings
def recursive_replace(self):
"""Recursively replace modules in the model as needed.
Currently, this replaces:
- GLUs with a fused `MergedColumnParallelLinear` + `...AndMul`
- Attention QKV projections with a fused `QKVParallelLinear` + split
- `nn.Linear` with vLLM's tensor parallel linear classes
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
- `nn.LayerNorm` with vLLM's `LayerNorm`
"""
tp_plan = self.model.tp_plan or {}
if not tp_plan and self.tp_group.world_size > 1:
tip = get_feature_request_tip(
self.model_config.model, self.model_config.trust_remote_code
)
logger.warning_once(
"%s does not define a tensor parallel plan. The Transformers modeling "
"backend will shard the model the best it can during graph fusion and "
"replicate the rest. This may be suboptimal or fail if the model does "
"not fuse cleanly. %s",
type(self.model).__name__,
tip,
)
# Prefix the patterns because we always start from `self.model`
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
# Detect fusable patterns once per module class (cached, so this is cheap)
fusers = Fusers(self.model, self.vllm_config)
vocab_embeddings = self._vocab_embeddings()
orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {}
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
"""Register a fused layer's mappings just before it is built."""
self.fusers.setdefault(prefix, []).append(fuser)
if (
isinstance(fuser, AttentionFuser)
and (index := fuser.layer_index(module)) is not None
):
if index in self.attention_fusers:
other, _ = self.attention_fusers[index]
raise ValueError(
f"Layer {index} dispatches attention from more than one "
f"module ({other} and {prefix}), which one vLLM attention "
"layer cannot serve."
)
self.attention_fusers[index] = (prefix, fuser)
orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix))
packed_modules_mapping = fuser.packed_modules_mapping
self.packed_modules_mapping.update(packed_modules_mapping)
if self.quant_config is not None:
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
new_module = child_module
qual_name = maybe_prefix(prefix, child_name)
if (
isinstance(module, nn.ModuleList)
and len(module) == self.text_config.num_hidden_layers
):
# Populate Eagle3 attrs
self._target_class = type(child_module)
layer_name = qual_name.removeprefix("model.")
self._layer_names[int(child_name)] = layer_name
# MTP weights should not be loaded into the base model
num_hidden_layers = self.text_config.num_hidden_layers
names = (
"n_predict", # Override from SpeculativeConfig
"num_nextn_predict_layers", # Most models
"mtp_num_hidden_layers", # Qwen 3.5
)
n_predict = getattr_iter(self.text_config, names, 0)
for i in range(num_hidden_layers, num_hidden_layers + n_predict):
mtp_prefix = f"{prefix}.{i}."
if mtp_prefix not in self.ignore_unexpected_prefixes:
self.ignore_unexpected_prefixes.append(mtp_prefix)
# Replace modules as needed
if isinstance(child_module, nn.Linear):
generator = (p for p in tp_plan if re.match(p, qual_name))
pattern = next(generator, None)
# Some weight loaders expect all linear layers to inherit
# LinearBase, so we set a default style which causes any
# unspecified layers to be replaced with ReplicatedLinear
style = tp_plan.get(pattern, "replicate")
new_module = replace_linear_class(
child_module, style, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
new_module = replace_conv_class(child_module)
elif child_module in vocab_embeddings:
new_module = replace_embedding_class(
child_module, self.quant_config, prefix=qual_name
)
elif isinstance(child_module, nn.LayerNorm):
new_module = replace_layernorm_class(child_module)
elif child_module_fusers := fusers[child_module]:
for fuser in child_module_fusers:
register_fusion(fuser, qual_name, child_module)
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
logger.info_once(fuser.info(child_name))
_recursive_replace(new_module, prefix=qual_name)
elif not isinstance(child_module, MoERunner):
# MoERunner can contain aliases of shared experts and gates,
# so we don't want to recurse into it and break weight loading.
_recursive_replace(child_module, prefix=qual_name)
if new_module is not child_module:
setattr(module, child_name, new_module)
log_replacement(qual_name, child_module, new_module)
_recursive_replace(self.model, prefix="model")
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
def _create_attention_instances(self, prefix: str = ""):
"""Create `Attention` instances to inform KV cache allocation."""
text_config = self.text_config
attn_cls = self._get_attn_cls()
# kv_lora_rank indicates that this is an MLA model
if getattr(text_config, "kv_lora_rank", None) is not None:
if attn_cls is MLAAttention:
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
else:
# MLA model not using MLAAttention: recompute head_size for full attn
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0)
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
self.model_config.model_arch_config.head_size = qk_head_dim
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
layer_types = getattr(text_config, "layer_types", None)
pp_rank = self.pp_group.rank_in_group
pp_size = self.pp_group.world_size
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
for i in range(start, end):
if i not in self.attention_fusers:
layer = (
f"{i} ({layer_types[i]})"
if layer_types and i < len(layer_types)
else str(i)
)
raise ValueError(
f"Layer {layer} does not dispatch through the Transformers "
"attention interface and vLLM has no other way to handle it."
)
attn_prefix, attn_fuser = self.attention_fusers[i]
attn_module = self.get_submodule(attn_prefix)
# `[i]` is the whole-model config unless the checkpoint is
# heterogeneous, in which case it is this layer's own geometry.
arch_config = self.model_config.model_arch_config[i]
num_heads = self.model_config.get_num_attention_heads(
self.parallel_config, arch_config
)
head_size = arch_config.head_size
if (scale := attn_fuser.scale(attn_module)) is None:
# Default to Llama scale if AttentionFuser couldn't identify it
scale = head_size**-0.5
kwargs = dict(
num_heads=num_heads,
scale=scale,
cache_config=self.cache_config,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
)
if attn_cls is MLAAttention:
mla_fuser = next(
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
None,
)
if mla_fuser is None:
raise ValueError(
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
"of the model did, so its latent KV cache cannot be allocated."
)
from vllm.model_executor.layers.attention.mla_attention import (
get_mla_dims,
)
dims = get_mla_dims(self.model_config)
kwargs.update(
qk_nope_head_dim=dims.qk_nope_head_dim,
qk_rope_head_dim=dims.qk_rope_head_dim,
v_head_dim=dims.v_head_dim,
q_lora_rank=dims.q_lora_rank,
kv_lora_rank=dims.kv_lora_rank,
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
)
else:
kwargs.update(
head_size=head_size,
num_kv_heads=self.model_config.get_num_kv_heads(
self.parallel_config, arch_config
),
logits_soft_cap=logits_soft_cap,
)
# Handle interleaved sliding window attention
if layer_types and layer_types[i] == "sliding_attention":
kwargs["per_layer_sliding_window"] = text_config.sliding_window
# Handle attention sinks
if (sinks := attn_fuser.sinks(attn_module)) is not None:
kwargs["sinks"] = sinks
attn_instance = attn_cls(**kwargs)
# Attach the instance to the module it serves, which is the only way it
# is in the module tree at all: a plain dict is invisible to
# `nn.Module.__setattr__`. Being in the tree runs its
# `process_weights_after_loading` and moves its KV-scale buffers with
# `model.to(device)`, and lets the attention interface read it off the
# module rather than index by `module.layer_idx`, an `int` that bakes
# layer identity into the traced graph and so costs one compiled
# artifact per layer.
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
def _get_attn_cls(self) -> type[AttentionLayerBase]:
"""Return the `Attention` class to use for this model's layers."""
# In encoder models, the attention layers will have `is_causal=False`
is_encoder = lambda module: not getattr(module, "is_causal", True)
has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
is_multimodal = lambda config: config != config.get_text_config()
# vLLM does not support encoder-decoder models, so if any encoder layer is
# found in a text only model, we assume the whole model is an encoder model
if has_encoder(self.model) and not is_multimodal(self.config):
return EncoderOnlyAttention
if self.model_config.use_mla:
if any(
isinstance(fuser, MLAFuser)
for fusers in self.fusers.values()
for fuser in fusers
):
return MLAAttention
logger.warning_once(
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
"MLA attention layers. Falling back to full attention with a padded "
"`value` head dimension."
)
os.environ["VLLM_MLA_DISABLE"] = "1"
return Attention
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
"""If a `parameter` is on the `meta` device, then its parent
`module` is the original module created by:
```python
with torch.device("meta"):
self.model: "PreTrainedModel" = AutoModel.from_config(...)
```
"""
dtype = dtype or self.model_config.dtype
device = self.device_config.device
def _init_parameters(module: nn.Module):
for name, param in module.named_parameters(recurse=False):
# Already on device, nothing to do
if param.device != torch.device("meta"):
continue
# Already a vLLM parameter, nothing to do
if hasattr(param, "weight_loader"):
continue
data = torch.empty_like(param.data, dtype=dtype, device=device)
setattr(module, name, nn.Parameter(data=data))
for child in module.children():
_init_parameters(child)
_init_parameters(module)
def keep_in_fp32(self, module: nn.Module):
"""Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would."""
if self.model_config.dtype not in (torch.float16, torch.bfloat16):
return
fragments = getattr(module, "_keep_in_fp32_modules_strict", None)
if not fragments:
return
pattern = re.compile("|".join(re.escape(f) for f in fragments))
for name, tensor in named_state(module):
if not hasattr(tensor, "weight_loader") and pattern.search(name):
tensor.data = tensor.data.to(torch.float32)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.get_input_embeddings()(input_ids)
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | IntermediateTensors:
if not self.pp_group.is_first_rank:
assert intermediate_tensors is not None
input_ids = None
inputs_embeds = intermediate_tensors["hidden_states"]
# Add batch dimension before entering Transformers model
if input_ids is not None and input_ids.ndim == 1:
# [seq_len] -> [1, seq_len]
input_ids = input_ids[None, ...]
if inputs_embeds is not None and inputs_embeds.ndim == 2:
# [seq_len, hidden_size] -> [1, seq_len, hidden_size]
inputs_embeds = inputs_embeds[None, ...]
if positions.ndim == 1:
# [seq_len] -> [1, seq_len]
positions = positions[None, ...]
# Transformers models expect either input_ids or inputs_embeds, but not both
if input_ids is not None and inputs_embeds is not None:
input_ids = None
outputs = self.model(
input_ids=input_ids,
inputs_embeds=inputs_embeds,
use_cache=False,
position_ids=positions,
return_dict=False,
**self._output_aux_hidden_states_kwargs,
**kwargs,
)
# Remove batch dimension after exiting Transformers model
hidden_states = outputs[0][0, ...]
if self._output_aux_hidden_states_kwargs:
aux_hidden_states = [x[0][0, ...] for x in outputs[1:]]
if not self.pp_group.is_last_rank:
return IntermediateTensors({"hidden_states": hidden_states})
if self._output_aux_hidden_states_kwargs and len(aux_hidden_states) > 0:
return hidden_states, aux_hidden_states
return hidden_states
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
loader = AutoWeightsLoader(
self,
ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
)
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
@staticmethod
def check_version(min_version: str, feature: str):
installed = Version(transformers.__version__)
required = Version(min_version)
if installed < required:
raise ImportError(
f"Transformers modeling backend requires transformers>={required} "
f"for {feature}, but got {installed}"
)
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
from transformers.utils.output_capturing import (
OutputRecorder,
maybe_install_capturing_hooks,
)
# The default value in PreTrainedModel is None
if self.model._can_record_outputs is None:
self.model._can_record_outputs = {}
target_class = self._target_class
for layer in layers:
# layer - 1 because we want the input to the layer
layer_name = self._layer_names[layer - 1]
layer_key = f"aux_hidden_state_{layer}"
aux_hidden_state_i = OutputRecorder(target_class, layer_name=layer_name)
self.model._can_record_outputs[layer_key] = aux_hidden_state_i
self._output_aux_hidden_states_kwargs[f"output_{layer_key}"] = True
# Ensure that the capture hooks are installed before dynamo traces the model
maybe_install_capturing_hooks(self.model)
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
num_layers = self.text_config.num_hidden_layers
return (2, num_layers // 2, num_layers - 3)
@@ -1,89 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixin for causal language models."""
from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.layers.logits_processor import LogitsProcessor
from vllm.model_executor.layers.vocab_parallel_embedding import (
ParallelLMHead,
VocabParallelEmbedding,
)
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
from .base import Base
if TYPE_CHECKING:
from vllm.config import VllmConfig
class CausalMixin(VllmModelForTextGeneration, Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip VllmModelForTextGeneration.__init__ and call the next class in MRO
super(VllmModelForTextGeneration, self).__init__(
vllm_config=vllm_config, prefix=prefix
)
tie_word_embeddings = self._get_tie_word_embeddings()
if self.pp_group.is_last_rank:
self.lm_head = ParallelLMHead(
self.text_config.vocab_size,
self.text_config.hidden_size,
quant_config=self.quant_config,
prefix=maybe_prefix(prefix, "lm_head"),
)
if tie_word_embeddings:
for module in self.model.get_input_embeddings().modules():
if isinstance(module, VocabParallelEmbedding):
self.lm_head = self.lm_head.tie_weights(module)
break
self.logits_processor = LogitsProcessor(
self.text_config.vocab_size,
scale=getattr(self.text_config, "logit_scale", 1.0),
soft_cap=getattr(self.text_config, "final_logit_softcapping", None),
)
else:
self.lm_head = PPMissingLayer()
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
lm_head_bias = set()
load_device = self.vllm_config.load_config.device or self.device_config.device
def auto_load_lm_head_bias(weights):
for name, weight in weights:
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
# load_weights is called outside vLLM's device context,
# so we must enter it again when registering the bias.
with torch.device(load_device):
self.lm_head._register_bias()
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
lm_head_bias.add(name)
else:
yield name, weight
return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias
def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
return logits
@@ -1,126 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser detection for the Transformers modeling backend."""
from collections import UserDict
from typing import TYPE_CHECKING, TypeVar
from cachetools import cached
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers import (
AttentionFuser,
BaseFuser,
GLUFuser,
MergedColumnParallelFuser,
MLAFuser,
PackedQKVFuser,
QKVFuser,
RewriteFuser,
RMSNormFuser,
)
from vllm.model_executor.models.transformers.fx_utils import trace
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
F = TypeVar("F", bound=BaseFuser)
def key(module: nn.Module) -> tuple:
"""Cache key for `get_fusers`. Considers module type and its immediate children."""
return (type(module), tuple(name for name, _ in module.named_children()))
FUSERS: tuple[type[BaseFuser], ...] = (
# Order these by priority, e.g. QKV would be preferred over MergedColumnParallel
MLAFuser,
GLUFuser,
QKVFuser,
PackedQKVFuser,
MergedColumnParallelFuser,
RMSNormFuser,
# Put fusers that don't redefine forward after this comment
AttentionFuser,
)
"""Every fuser, in priority order: those that redefine the forward first, then those
that leave it alone. A new fuser is added here."""
@cached(cache={}, key=key)
def get_fusers(module: nn.Module) -> list[BaseFuser]:
"""Every fuser that could apply to `module`'s class (cached), in `FUSERS` order."""
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
# leaf module (raw tensor math, no submodules). Nothing else can match, and
# tracing is skipped for it.
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
is_leaf = next(module.children(), None) is None
graph = trace(module) if n_linear >= 2 or is_leaf else None
# Every redefining candidate that matches is cached, not just the first:
# instances of this class can be shaped heterogeneously, so which one is
# usable is decided per instance in Fusers.__getitem__, not here. A forward
# can only be rewritten from a trace of it, so each starts fresh regardless.
fusers: list[BaseFuser] = []
for fuser_cls in FUSERS:
if fuser_cls.redefines_forward and graph is None:
continue
if (fuser := fuser_cls.match(graph, module)) is None:
continue
if isinstance(fuser, RewriteFuser):
try:
fuser.update_forward(module)
except Exception as exc:
logger.debug(
"Attempted to fuse %s using %s but failed "
"to update its forward method: %s",
type(module),
fuser_cls.__name__,
exc,
)
continue
fusers.append(fuser)
# A norm we could not match structurally is left unfused; flag likely misses.
if (
module.__class__.__name__.endswith("RMSNorm")
and graph is not None
and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
):
logger.warning_once(
"%s looks like an RMSNorm but its computation did not match the "
"expected pattern, so it was left unfused.",
module.__class__.__name__,
)
return fusers
def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
"""The first `fuser_cls` that applies to `module`, if one does."""
return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
class Fusers(UserDict):
"""Mapping from module class and shape to fusers, for all fusable modules."""
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
self.vllm_config = vllm_config
super().__init__({key(m): get_fusers(m) for m in model.modules()})
def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
"""The fusers this instance can take, in the order to apply them."""
chosen: list[BaseFuser] = []
redefined = False
for fuser in self.data.get(key(m), ()):
# Only one candidate may rewrite m's forward; take the first
# (highest-priority) one that validates for this instance.
if fuser.redefines_forward and redefined:
continue
if not fuser.validate(m, self.vllm_config):
continue
redefined = redefined or fuser.redefines_forward
chosen.append(fuser)
return chosen
@@ -1,33 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Concrete fusers for the Transformers modeling backend."""
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
from vllm.model_executor.models.transformers.fusers.base import (
BaseFuser,
RewriteFuser,
StackedFuser,
)
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.merged_column import (
MergedColumnParallelFuser,
)
from vllm.model_executor.models.transformers.fusers.mla import MLAFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
__all__ = [
"AttentionFuser",
"BaseFuser",
"RewriteFuser",
"StackedFuser",
"GLUFuser",
"MergedColumnParallelFuser",
"MLAFuser",
"MoEBlockFuser",
"PackedQKVFuser",
"QKVFuser",
"RMSNormFuser",
]
@@ -1,162 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Attention fuser: the module that dispatches to the attention interface."""
import ast
import inspect
import textwrap
from collections.abc import Callable
from dataclasses import dataclass
from functools import cache
from typing import TYPE_CHECKING, ClassVar
import torch
from torch import fx, nn
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.utils import set_weight_attrs
if TYPE_CHECKING:
from vllm.config import VllmConfig
VLLM_ATTN_IMPL = "vllm"
VLLM_MLA_ATTN_IMPL = "vllm_mla"
def _is_interface_lookup(node: ast.expr | None) -> bool:
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
if isinstance(node, ast.Call):
node = node.func
if not isinstance(node, (ast.Attribute, ast.Subscript)):
return False
return (
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
)
@cache
def interface_call(forward: Callable) -> ast.Call | None:
"""The attention interface call in `forward`, if it makes exactly one."""
try:
source = inspect.getsource(inspect.unwrap(forward))
tree = ast.parse(textwrap.dedent(source))
except (AttributeError, OSError, SyntaxError, TypeError):
return None
# Find the names of the local variables that read from the interface lookup
names: set[str] = set()
for node in ast.walk(tree):
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
continue
if not _is_interface_lookup(node.value):
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
names.update(t.id for t in targets if isinstance(t, ast.Name))
# Find the calls to those names, and return the one if there is exactly one
calls = [
node
for node in ast.walk(tree)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Name)
and node.func.id in names
]
return calls[0] if len(calls) == 1 else None
def _resolve(node: ast.expr, module: nn.Module) -> object:
"""The value of `node` on `module`, for literals and `self.<attr>`."""
if isinstance(node, ast.Constant):
return node.value
if (
isinstance(node, ast.Attribute)
and isinstance(node.value, ast.Name)
and node.value.id == "self"
):
return getattr(module, node.attr, None)
return None
@dataclass
class AttentionFuser(BaseFuser):
"""A module that dispatches through the Transformers attention interface."""
redefines_forward: ClassVar[bool] = False
source_cls: str
"""Class of the HF module that dispatches (for logging)."""
scale_expr: ast.expr | None = None
"""Source of the `scaling=` the module hands the interface, if it hands one."""
s_aux_expr: ast.expr | None = None
"""Source of the `s_aux=` the module hands the interface, if it hands one."""
def info(self, name: str) -> str:
return f"Found: {name} ({self.source_cls}) -> attention interface"
@classmethod
def match(
cls, graph: fx.Graph | None, module: nn.Module
) -> "AttentionFuser | None":
if (call := interface_call(type(module).forward)) is None:
return None
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
scale_expr = scaling[0] if len(scaling) == 1 else None
s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"]
s_aux_expr = s_aux[0] if len(s_aux) == 1 else None
return cls(
source_cls=type(module).__name__,
scale_expr=scale_expr,
s_aux_expr=s_aux_expr,
)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether `module` will actually dispatch to vLLM."""
config = getattr(module, "config", None)
# Only patched in the text config, this excludes attention based mm encoders
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
if (sinks := self.sinks(module)) is not None:
assert isinstance(self.s_aux_expr, ast.Attribute)
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
device = vllm_config.device_config.device
data = torch.empty(size, dtype=sinks.dtype, device=device)
sinks_param = nn.Parameter(data, requires_grad=False)
set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)})
setattr(module, self.s_aux_expr.attr, sinks_param)
return module
def layer_index(self, module: nn.Module) -> int | None:
"""The layer `module` computes attention for, if it declares one."""
layer_idx = getattr(module, "layer_idx", None)
return layer_idx if isinstance(layer_idx, int) else None
def scale(self, module: nn.Module) -> float | None:
"""The softmax scale `module` passes to the interface, or `None`."""
if self.scale_expr is None:
return None
scale = _resolve(self.scale_expr, module)
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
expression = ast.unparse(self.scale_expr)
raise ValueError(
f"Cannot resolve attention scaling expression {expression!r} in "
f"{type(module).__name__}."
)
return float(scale)
def sinks(self, module: nn.Module) -> nn.Parameter | None:
"""The per-head sink tensor `module` passes to the interface, or `None`."""
if self.s_aux_expr is None:
return None
s_aux = _resolve(self.s_aux_expr, module)
if not isinstance(s_aux, (nn.Parameter, type(None))):
expression = ast.unparse(self.s_aux_expr)
raise ValueError(
f"Cannot resolve attention s_aux expression {expression!r} in "
f"{type(module).__name__}."
)
return s_aux
@@ -1,250 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Base classes for the Transformers backend fusers."""
import ast
import types
from abc import ABC, abstractmethod
from collections.abc import Callable, Iterable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.model_executor.models.transformers.fx_utils import (
aliasing_names,
bypass_existence_guard,
replace_expr,
self_call_and_refs,
written_names,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
@dataclass
class BaseFuser(ABC):
"""A detected fusion and how to apply it.
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
then applies the fusion to an instance in `recursive_replace`, returning the
module to install in its place.
"""
redefines_forward: ClassVar[bool] = True
"""Whether `fuse` gives the module a different forward,
by rewriting its source or by returning a different module in its place."""
@abstractmethod
def info(self, name: str) -> str:
"""A human-readable description of the fusion at `name`, for logging."""
@classmethod
@abstractmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "BaseFuser | None":
"""Match the pattern in `graph`, returning a fuser if found."""
@abstractmethod
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Whether this fuser can be applied to this `module` instance."""
@abstractmethod
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Apply the fusion to an already-validated `module`, returning the
module to install in its place (mutated in place, or freshly built)."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
(none unless it stacks weights)."""
return {}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`packed_modules_mapping` entries this fuser contributes (none unless
it stacks weights)."""
return {}
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
"""The head size of `module`, which head counts are derived from.
Prefer the module's own `head_dim`, which Transformers sets per instance:
the model-wide value is the largest head size across layers, so on a
heterogeneous checkpoint it would divide out the wrong number of heads, and
it is the text head size, so it does not describe a vision tower at all.
"""
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
def local_output_sizes(merged_name: str) -> str:
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
merged = f"self.{merged_name}"
return f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
@dataclass
class RewriteFuser(BaseFuser):
"""A fuser that rewrites the module's forward and rebinds it.
`match` and `update_forward` analyse the class once; `fuse` swaps the
submodules and binds the compiled forward on an instance in place, so it
keeps its class and any attribute the fusion does not consume.
"""
source_cls: str
"""Class of the HF module the fused projections belonged to (for logging)."""
fused_forward: Callable = field(init=False, repr=False)
"""The compiled rewritten forward, set by `update_forward`."""
@abstractmethod
def update_forward(self, module: nn.Module) -> None:
"""Rewrite and compile `type(module)`'s forward source.
Raises if the source does not admit the rewrite (fusion is then skipped).
"""
@abstractmethod
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace `module`'s submodules with their vLLM equivalents."""
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
Builds the merged submodule and binds the compiled forward."""
self.update_attrs(module, prefix, vllm_config)
module.forward = types.MethodType(self.fused_forward, module)
return module
@dataclass
class StackedFuser(RewriteFuser):
"""A fuser that merges sibling projections into one stacked linear and
rewrites the forward to call it."""
merged_cls_name: ClassVar[str]
"""Name of the vLLM class the merged projection becomes (for logging)."""
@property
@abstractmethod
def merged_name(self) -> str:
"""Attribute name of the merged module created by `update_attrs`."""
def info(self, name: str) -> str:
sources = " + ".join(shard for shard, _ in self.shards)
return (
f"Fused: {sources} ({name}: {self.source_cls}) -> "
f"{self.merged_name} ({self.merged_cls_name})"
)
@property
@abstractmethod
def shards(self) -> list[tuple[str, ShardId]]:
"""Each projection's original name and its shard id in the merged module.
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
so only this exact layer is remapped, never a same-named projection
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
merged = maybe_prefix(prefix, self.merged_name)
return {
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
}
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
"""`{merged_name: [projection names]}` so quantization can unpack the
fused layer into its per-shard configs."""
return {self.merged_name: [name for name, _ in self.shards]}
def _unguarded_calls(
self, funcdef: ast.FunctionDef, names: Iterable[str]
) -> list[ast.Call]:
"""One `self.<name>(arg)` call per projection, existence guards folded.
`update_attrs` deletes the projections it merges, so any reference to one beyond
its call site must be a guard on its existence; folding those to their constant
value keeps the rewritten forward off a name that no longer exists. A reference
that is not such a guard raises, so fusion is skipped."""
calls = []
for name in names:
call, refs = self_call_and_refs(funcdef, name)
for ref in refs:
bypass_existence_guard(funcdef, ref, name)
calls.append(call)
return calls
def _check_input_stable(
self,
funcdef: ast.FunctionDef,
module: nn.Module,
calls: list[ast.Call],
block: list[ast.stmt],
indices: list[int],
) -> None:
"""Raise unless hoisting the merged GEMM preserves what it reads.
Fusing moves every projection to one call at `min(indices)`, so the
merged GEMM reads the input once, up front, where the last of `calls`
would have read it later. That holds only if nothing in between changes
the input, and a change need not name it: writing any view that shares
its storage changes it too. Both halves of the check over-approximate,
since a false hit costs a fusion while a miss returns wrong numbers.
"""
arg_names = {
node.id
for node in ast.walk(calls[0].args[0])
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
}
# Protect the names the input reads and anything that may alias them.
tracked = aliasing_names(funcdef, arg_names, module)
# Any of them rebound, written through, or passed to a call that writes
# its argument would leave the hoisted GEMM reading a different value.
region = block[min(indices) : max(indices) + 1]
if tracked & written_names(region):
raise ValueError("projection input is rebound or mutated before all calls")
def _splice_merged_split(
self,
funcdef: ast.FunctionDef,
calls: list[ast.Call],
block: list[ast.stmt],
index: int,
) -> None:
"""Insert `temps = self.<merged_name>(arg).split(sizes, -1)` at
`block[index]` and replace each of `calls` with its temp name.
`calls` must share one input argument; `block[index]` must be where
they are (or would be) evaluated. Raises if a generated temporary
would shadow an existing name in `funcdef`.
"""
temps = [f"_vllm_merged_{i}" for i in range(len(calls))]
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
if names & set(temps):
raise ValueError("fused temporaries would shadow existing names")
targets = ", ".join(temps)
sections = local_output_sizes(self.merged_name)
source = f"{targets} = self.{self.merged_name}(__arg__).split({sections}, -1)"
assign = ast.parse(source).body[0]
arg = next(
node
for node in ast.walk(assign)
if isinstance(node, ast.Name) and node.id == "__arg__"
)
replace_expr(assign, arg, calls[0].args[0])
ast.copy_location(assign, block[index])
block.insert(index, assign)
for call, temp in zip(calls, temps):
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
@@ -1,191 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""GLU projection fuser: `act(gate(x)) * up(x)` -> a fused gate/up linear."""
import ast
import operator
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from transformers.activations import ACT2CLS
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
from vllm.model_executor.models.transformers.fusers.merged_column import (
MergedColumnParallelFuser,
)
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
find_node,
is_linear,
peel,
recover_forward,
replace_expr,
single_self_call,
)
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
CLS2ACT: dict[type, list[str]] = {}
for _act_name, _act_cls in ACT2CLS.items():
if isinstance(_act_cls, tuple):
_act_cls = _act_cls[0]
CLS2ACT.setdefault(_act_cls, []).append(_act_name)
ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys())
@dataclass
class GLUFuser(MergedColumnParallelFuser):
"""Fuser for the GLU pattern `act(gate(x)) * up(x)`."""
act_name: str
down_name: str | None
merged_name: ClassVar[str] = "gate_up_proj"
@property
def gate_name(self) -> str:
return self.linear_names[0]
@property
def up_name(self) -> str:
return self.linear_names[1]
@classmethod
def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool:
"""Is node `act(gate(x))` where `gate` is linear and `act` is not linear."""
return (
node.op == "call_module"
and not is_linear(node, module)
and len(node.args) == 1
and isinstance(node.args[0], fx.Node)
and is_linear(node.args[0], module)
)
@classmethod
def _get_glu_nodes(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node, fx.Node] | None:
"""Search graph for the GLU pattern `act(gate(x)) * up(x)`."""
for mul in graph.nodes:
if (
mul.op == "call_function"
and mul.target == operator.mul
and len(mul.args) == 2
and all(isinstance(arg, fx.Node) for arg in mul.args)
):
a, b = mul.args
if cls._is_act_of_gate(a, module) and is_linear(b, module):
act, gate, up = a, a.args[0], b
elif cls._is_act_of_gate(b, module) and is_linear(a, module):
act, gate, up = b, b.args[0], a
else:
continue
if (
all(len(args) == 1 for args in (gate.args, up.args))
and isinstance(x := gate.args[0], fx.Node)
and x is up.args[0]
):
return act, gate, up, mul
return None
@staticmethod
def _get_act_and_mul_name(act: nn.Module) -> str | None:
"""Get the name of `act` if it has an `...AndMul` equivalent."""
for name in CLS2ACT.get(type(act), []):
if name in ACT_AND_MUL_NAMES:
return name
# nn.GELU is not in ACT2CLS, but could be in model code
if type(act) is nn.GELU:
return "gelu_pytorch_tanh" if act.approximate == "tanh" else "gelu"
return None
@classmethod
def _get_act_and_mul(cls, act: nn.Module) -> nn.Module:
"""Get the `...AndMul` equivalent of a Transformers activation module."""
if name := cls._get_act_and_mul_name(act):
return get_act_and_mul_fn(name)
raise ValueError(f"No AndMul equivalent for {type(act)}")
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "GLUFuser | None":
if (glu_nodes := cls._get_glu_nodes(graph, module)) is None:
return None
act_node, gate_node, up_node, mul_node = glu_nodes
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
down_node = find_node(graph, predicate)
return cls(
source_cls=type(module).__name__,
act_name=act_node.target,
linear_names=(gate_node.target, up_node.target),
down_name=down_node.target if down_node is not None else None,
)
def update_forward(self, module: nn.Module) -> None:
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
funcdef, fn = recover_forward(type(module))
act_call = single_self_call(funcdef, self.act_name)
calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name))
gate_call, up_call = calls
if act_call.args[0] is not gate_call:
raise ValueError("activation does not directly wrap the gate")
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
raise ValueError("gate and up inputs are written differently")
muls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.BinOp)
and isinstance(node.op, ast.Mult)
and {id(node.left), id(node.right)} == {id(act_call), id(up_call)}
]
if len(muls) != 1:
raise ValueError("no multiply of the activation and up projection")
# act(gate(x)) * up(x) -> act(gate_up(x))
assert isinstance(gate_call.func, ast.Attribute)
gate_call.func.attr = self.merged_name
replace_expr(funcdef, muls[0], act_call)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
if not super().validate(module, vllm_config):
return False
# An AndMul kernel splits its input in half, so gate and up must match.
gate = module.get_submodule(self.gate_name)
up = module.get_submodule(self.up_name)
if gate.out_features != up.out_features:
logger.debug("%s and %s differ in size; skipping fusion", gate, up)
return False
act = module.get_submodule(self.act_name)
if self._get_act_and_mul_name(act) is None:
logger.debug("%s has no AndMul equivalent; skipping fusion", act)
return False
return True
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
super().update_attrs(module, prefix, vllm_config)
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
setattr(module, self.act_name, act_fn)
# If there is a down projection, we know it must be rowwise.
if self.down_name is not None:
down_prefix = maybe_prefix(prefix, self.down_name)
down = module.get_submodule(self.down_name)
new_down = replace_linear_class(
down, "rowwise", vllm_config.quant_config, prefix=down_prefix
)
setattr(module, self.down_name, new_down)
log_replacement(down_prefix, down, new_down)
@@ -1,145 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Fuser for parallel linear projections."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fx_utils import (
block_chain,
compile_forward,
is_linear,
recover_forward,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class MergedColumnParallelFuser(StackedFuser):
"""Fuser for merging column-parallel linear projections."""
linear_names: tuple[str, ...]
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
@property
def merged_name(self) -> str:
"""Programmatic name for the merged projection, based on the original names."""
# len is used to disambiguate names like `a_b` + `c` vs `a` + `b_c`
parts = "_".join(f"{len(name)}_{name}" for name in self.linear_names)
return f"merged_proj_{parts}"
@property
def shards(self) -> list[tuple[str, ShardId]]:
return [(name, index) for index, name in enumerate(self.linear_names)]
@staticmethod
def _names(group: list[fx.Node]) -> tuple[str, ...] | None:
names = tuple(str(node.target) for node in group)
if len(set(names)) != len(names) or any("." in name for name in names):
return None
return names
@classmethod
def match(
cls, graph: fx.Graph, module: nn.Module
) -> "MergedColumnParallelFuser | None":
"""Fuse the module's sibling linears when there is only one such group."""
by_input: dict[fx.Node, list[fx.Node]] = {}
for node in graph.nodes:
if (
is_linear(node, module)
and len(node.args) == 1
and not node.kwargs
and isinstance(node.args[0], fx.Node)
# Like QKVFuser/PackedQKVFuser: this fuser has no head or
# TP-replication awareness, so it must not absorb a QKV-shaped
# pattern (e.g. behind a norm) a more specific fuser declined.
and node.args[0].op == "placeholder"
):
by_input.setdefault(node.args[0], []).append(node)
# A group whose members are not distinct direct children is dropped:
# the source rewrite addresses each projection as `self.<name>` once.
groups = [
(group, names)
for group in by_input.values()
if len(group) >= 2 and (names := cls._names(group)) is not None
]
if len(groups) > 1:
logger.debug(
"%s has %d fusable sibling-linear groups; skipping fusion "
"since which one to merge is ambiguous",
type(module).__name__,
len(groups),
)
if len(groups) != 1:
return None
_, names = groups[0]
fuser = cls(source_cls=type(module).__name__, linear_names=names)
return None if hasattr(module, fuser.merged_name) else fuser
def update_forward(self, module: nn.Module) -> None:
"""Replace the parallel calls with one merged call and split."""
funcdef, fn = recover_forward(type(module))
calls = self._unguarded_calls(funcdef, self.linear_names)
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
raise ValueError("parallel linears read different inputs")
chains = [block_chain(funcdef.body, call) for call in calls]
if any(not chain for chain in chains):
raise ValueError("parallel linear calls not found in the function body")
blocks = [chain[-1] for chain in chains]
if len(set(id(block) for block, _ in blocks)) != 1:
raise ValueError("parallel linear calls are in different blocks")
block = blocks[0][0]
indices = [index for _, index in blocks]
index = min(indices)
self._check_input_stable(funcdef, module, calls, block, indices)
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
self._splice_merged_split(funcdef, calls, block, index)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Check that the parallel linears are compatible for merging."""
linear_layers = [module.get_submodule(name) for name in self.linear_names]
tp_size = vllm_config.parallel_config.tensor_parallel_size
return (
len(linear_layers) >= 2
and len(set(linear.in_features for linear in linear_layers)) == 1
and len(set(linear.bias is None for linear in linear_layers)) == 1
and all(linear.out_features % tp_size == 0 for linear in linear_layers)
)
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
"""Replace the module's parallel linears with one merged projection."""
linear_modules = [module.get_submodule(name) for name in self.linear_names]
merged = MergedColumnParallelLinear(
input_size=linear_modules[0].in_features,
output_sizes=[linear.out_features for linear in linear_modules],
bias=linear_modules[0].bias is not None,
quant_config=vllm_config.quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
)
setattr(module, self.merged_name, merged)
for name in self.linear_names:
delattr(module, name)
logger.debug(
"%s -> %s: %s",
", ".join(f"{n}: {m}" for n, m in zip(self.linear_names, linear_modules)),
self.merged_name,
merged,
)
@@ -1,303 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MLA fuser: adapt a Transformers MLA attention module for vLLM's MLA layer."""
import ast
import inspect
import textwrap
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
downstream_linear,
is_linear,
recover_forward,
replace_expr,
returned_linear,
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
Style,
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
def _consumes_placeholder(node: fx.Node) -> bool:
"""Whether `node` is a linear applied directly to a `forward` input."""
return (
len(node.args) == 1
and isinstance(node.args[0], fx.Node)
and node.args[0].op == "placeholder"
)
def _norm_size(norm: nn.Module) -> int:
weight = getattr(norm, "weight", None)
return weight.numel() if weight is not None else -1
def _is_rms_norm(module: nn.Module) -> bool:
"""Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher."""
# Go via `get_fuser` because it caches
from vllm.model_executor.models.transformers.fuser import get_fuser
return get_fuser(module, RMSNormFuser) is not None
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
"""Index in `funcdef.body` of the top-level statement containing `node`."""
for index, stmt in enumerate(funcdef.body):
if any(child is node for child in ast.walk(stmt)):
return index
raise ValueError("node is not in the function body")
def _single_expand_call(
funcdef: ast.FunctionDef, module: nn.Module, kv_b_proj_name: str
) -> ast.Call:
"""The KV expansion call `self.<method>(kv_c_normed, k_pe)`: the unique
two-argument call to a method of `module` with the expansion's signature.
The expansion is `kv_c_normed, k_pe -> key, value`: two inputs, every
`return` a pair, and it applies `kv_b_proj` -- the projection `MLAAttention`
owns after the bypass, absorbed into the attention computation.
"""
def is_expansion_method(name: str) -> bool:
method = getattr(type(module), name, None)
if method is None:
return False
code = getattr(inspect.unwrap(method), "__code__", None)
if (
code is None
or code.co_argcount != 3 # self, kv_c_normed, k_pe
or kv_b_proj_name not in code.co_names
):
return False
try:
tree = ast.parse(textwrap.dedent(inspect.getsource(method)))
except (OSError, TypeError):
return False
returns = [node for node in ast.walk(tree) if isinstance(node, ast.Return)]
return bool(returns) and all(
isinstance(ret.value, ast.Tuple) and len(ret.value.elts) == 2
for ret in returns
)
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and isinstance(node.func.value, ast.Name)
and node.func.value.id == "self"
and len(node.args) == 2
and not node.keywords
and is_expansion_method(node.func.attr)
]
if len(calls) != 1:
raise ValueError("expected exactly one KV expansion method call")
return calls[0]
@dataclass
class MLAFuser(StackedFuser):
"""Fuser for the MLA attention pattern."""
q_proj_name: str | None
q_a_proj_name: str | None
q_a_layernorm_name: str | None
q_b_proj_name: str | None
kv_a_proj_name: str
kv_a_layernorm_name: str
kv_b_proj_name: str
o_proj_name: str | None
merged_name: ClassVar[str] = "fused_qkv_a_proj"
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
def info(self, name: str) -> str:
info_str = (
f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)"
)
if self.q_a_proj_name is not None:
info_str += "; " + super().info(name).removeprefix("Fused: ")
return info_str
@property
def shards(self) -> list[tuple[str, ShardId]]:
"""`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection."""
if self.q_a_proj_name is None:
return []
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
@property
def packed_modules_mapping(self) -> dict[str, list[str]]:
if self.q_a_proj_name is None:
return {}
return super().packed_modules_mapping
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None":
# Find all `rms_norm(linear(placeholder))` chains.
chains = []
for node in graph.nodes:
if node.op != "call_module" or is_linear(node, module) or not node.args:
continue
source = upstream_linear(node.args[0], module)
if source is None or not _consumes_placeholder(source):
continue
if not _is_rms_norm(module.get_submodule(node.target)):
continue
chains.append((source, node))
# Tell the chains apart by width.
def is_kv_a(chain) -> bool:
source, rms_norm = chain
source_mod = module.get_submodule(source.target)
rms_norm_mod = module.get_submodule(rms_norm.target)
return source_mod.out_features != _norm_size(rms_norm_mod)
kv_a_chains = [chain for chain in chains if is_kv_a(chain)]
q_a_chains = [chain for chain in chains if not is_kv_a(chain)]
# Exactly one KV chain is MLA's signature. The query chain is optional.
if len(kv_a_chains) != 1 or len(q_a_chains) > 1:
return None
kv_a_proj, kv_a_layernorm = kv_a_chains[0]
# Linear children claimed for a role so far; the rest resolve by elimination.
claimed_linears = {kv_a_proj.target}
q_proj_name = q_a_proj = q_a_layernorm = q_b_proj = None
if q_a_chains:
# Find `q_b_proj(q_a_layernorm(q_a_proj(placeholder)))`.
q_a_proj, q_a_layernorm = q_a_chains[0]
q_b_proj = downstream_linear(q_a_layernorm, module)
if q_b_proj is None:
return None
claimed_linears |= {q_a_proj.target, q_b_proj.target}
else:
# Find `q_proj(placeholder)`.
placeholder_linears = {
node.target
for node in graph.nodes
if is_linear(node, module) and _consumes_placeholder(node)
}
if len(q_proj_candidates := placeholder_linears - claimed_linears) != 1:
return None
q_proj_name = next(iter(q_proj_candidates))
claimed_linears.add(q_proj_name)
# Find `kv_b_proj(kv_a_layernorm(...))`.
kv_b_proj = downstream_linear(kv_a_layernorm, module)
if kv_b_proj is None or kv_b_proj.target in claimed_linears:
return None
claimed_linears.add(kv_b_proj.target)
# Find `o_proj` if it is returned by the forward graph.
o_proj_name = returned_linear(graph, module)
if o_proj_name in claimed_linears:
o_proj_name = None
return cls(
source_cls=type(module).__name__,
q_proj_name=q_proj_name,
q_a_proj_name=q_a_proj.target if q_a_proj else None,
q_a_layernorm_name=q_a_layernorm.target if q_a_layernorm else None,
q_b_proj_name=q_b_proj.target if q_b_proj else None,
kv_a_proj_name=kv_a_proj.target,
kv_a_layernorm_name=kv_a_layernorm.target,
kv_b_proj_name=kv_b_proj.target,
o_proj_name=o_proj_name,
)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
return vllm_config.model_config.use_mla
def update_forward(self, module: nn.Module) -> None:
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
attention interface unexpanded."""
funcdef, fn = recover_forward(type(module))
if (q_a_proj_name := self.q_a_proj_name) is not None:
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
# The fused call is inserted at the top-level statement preceding both.
names = [q_a_proj_name, self.kv_a_proj_name]
calls = self._unguarded_calls(funcdef, names)
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
raise ValueError("down-projections read different inputs")
indices = [_top_level_index(funcdef, call) for call in calls]
self._check_input_stable(funcdef, module, calls, funcdef.body, indices)
self._splice_merged_split(funcdef, calls, funcdef.body, min(indices))
# Transformers expands the latent into full key/value in a dedicated method.
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
# so replace the expansion call with its own arguments so `kv_c_normed, k_pe`
# flow to the interface in place of `key, value`.
expand_call = _single_expand_call(funcdef, module, self.kv_b_proj_name)
replace_expr(
funcdef, expand_call, ast.Tuple(elts=list(expand_call.args), ctx=ast.Load())
)
self.fused_forward = compile_forward(funcdef, fn)
def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"):
quant_config = vllm_config.quant_config
def replace_linear_by_name(name: str, style: Style):
linear = module.get_submodule(name)
replacement = replace_linear_class(
linear, style, quant_config, prefix=maybe_prefix(prefix, name)
)
setattr(module, name, replacement)
log_replacement(maybe_prefix(prefix, name), linear, replacement)
if (q_a_proj_name := self.q_a_proj_name) is not None:
# `match` sets the q-LoRA names together, or none of them
assert self.q_b_proj_name is not None
q_a = module.get_submodule(q_a_proj_name)
kv_a = module.get_submodule(self.kv_a_proj_name)
merged = MergedColumnParallelLinear(
input_size=q_a.in_features,
output_sizes=[q_a.out_features, kv_a.out_features],
bias=q_a.bias is not None,
quant_config=quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
disable_tp=True,
)
logger.debug(
"%s: %s, %s: %s -> %s: %s",
q_a_proj_name,
q_a,
self.kv_a_proj_name,
kv_a,
self.merged_name,
merged,
)
setattr(module, self.merged_name, merged)
# The rewritten forward calls the merged projection instead.
delattr(module, q_a_proj_name)
delattr(module, self.kv_a_proj_name)
replace_linear_by_name(self.q_b_proj_name, "colwise")
else:
assert self.q_proj_name is not None
replace_linear_by_name(self.kv_a_proj_name, "replicate")
replace_linear_by_name(self.q_proj_name, "colwise")
replace_linear_by_name(self.kv_b_proj_name, "colwise")
# MLAAttention calls kv_b_proj and expects vLLM's default return_bias=True
module.get_submodule(self.kv_b_proj_name).return_bias = True
if self.o_proj_name is not None:
replace_linear_by_name(self.o_proj_name, "rowwise")
@@ -1,439 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
import ast
import functools
import inspect
import textwrap
import types
from collections.abc import Iterable, Iterator
from dataclasses import dataclass
import torch
from torch import fx, nn
from typing_extensions import TypeIs
from vllm.distributed import tensor_model_parallel_all_gather
from vllm.logger import init_logger
from vllm.model_executor.layers.fused_moe import GateLinear
from vllm.model_executor.layers.linear import ReplicatedLinear
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
is_complete,
is_op,
output_value,
peel,
trace,
)
from vllm.model_executor.models.transformers.utils import named_state
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
logger = init_logger(__name__)
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
"""`return` statements in `node`'s own scope, not in nested functions."""
stack = list(ast.iter_child_nodes(node))
while stack:
child = stack.pop()
if isinstance(child, ast.Return):
yield child
elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
stack.extend(ast.iter_child_nodes(child))
def _returns_tuple(cls: type[nn.Module]) -> bool:
"""Does `cls.forward()` return a tuple?"""
try:
source = textwrap.dedent(inspect.getsource(inspect.unwrap(cls.forward)))
forward = ast.parse(source).body[0]
except (OSError, SyntaxError, TypeError, IndexError):
return True
# Names bound to a tuple literal, e.g. `out = hidden, logits` then `return out`.
tuple_names = {
target.id
for node in ast.walk(forward)
if isinstance(node, ast.Assign) and isinstance(node.value, ast.Tuple)
for target in node.targets
if isinstance(target, ast.Name)
}
def yields_tuple(value: ast.expr | None) -> bool:
if isinstance(value, ast.Tuple):
return True
if isinstance(value, ast.Name):
return value.id in tuple_names
if isinstance(value, ast.IfExp):
return yields_tuple(value.body) or yields_tuple(value.orelse)
return False
return any(yields_tuple(ret.value) for ret in _own_returns(forward))
def _is_scalar_gate(module: nn.Module) -> bool:
"""A linear projecting to a single logit (the shared-expert sigmoid gate)."""
weight = getattr(module, "weight", None)
return (
isinstance(module, nn.Linear)
and weight is not None
and weight.ndim == 2
and weight.shape[0] == 1
)
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
"""The floating dtype `nodes` cast to, if any.
Computations that must run in higher precision say so in their forward (e.g.
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
even when the config does not name it."""
for node in nodes:
if node.op not in ("call_method", "call_function"):
continue
name = str(node.target).rsplit(".", 1)[-1]
if name == "float":
return torch.float32
if name in ("to", "type"):
for arg in (*node.args[1:], *node.kwargs.values()):
if isinstance(arg, torch.dtype) and arg.is_floating_point:
return arg
return None
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
"""Returns the set of nodes reachable from `node` by following `key` edges."""
seen: set[fx.Node] = set()
stack = [node]
while stack:
n = stack.pop()
if n in seen:
continue
seen.add(n)
stack.extend(getattr(n, key))
return seen
def _renormalizes(topk: fx.Node) -> bool:
"""Are the top-k weights divided by their own sum?
Every HF router spells renormalization as a division by a sum over the
selection, optionally offset by an epsilon."""
downstream = _reaches(topk, "users")
for node in downstream:
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
continue
denominator = node.args[1]
if not isinstance(denominator, fx.Node):
continue
cone = _reaches(denominator, "all_input_nodes")
if any(is_op(n, "sum") and n in downstream for n in cone):
return True
return False
def _unwrap(node: object, moe_block: nn.Module) -> object:
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
while True:
node = peel(node)
if (
is_op(node, "reshape")
or is_op(node, "view")
or (
isinstance(node, fx.Node)
and node.op == "call_module"
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
)
):
node = node.args[0]
else:
return node
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
"""Is `node` a child module of `moe_block` called on just `x`?"""
return (
isinstance(node, fx.Node)
and node.op == "call_module"
and len(node.args) == 1
and not node.kwargs
and _unwrap(node.args[0], moe_block) is x
)
class SharedExpertMLP(nn.Module):
"""Wraps an HF shared expert, applying the output gating it is paired with."""
def __init__(self, shared_experts: nn.Module, gate: nn.Module | None = None):
super().__init__()
self.shared_experts = shared_experts
self.gate = gate
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
out = self.shared_experts(hidden_states)
if self.gate is not None:
out = torch.sigmoid(self.gate(hidden_states)[0]) * out
return out
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
"""Standard MoE block forward.
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
orig_shape = hidden_states.shape
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
num_tokens = hidden_states.shape[0]
is_sequence_parallel = self.experts.moe_config.is_sequence_parallel
if is_sequence_parallel:
hidden_states = sequence_parallel_chunk(hidden_states)
out = self.experts(hidden_states, router_logits=hidden_states)
if is_sequence_parallel:
out = tensor_model_parallel_all_gather(out, 0)[:num_tokens]
return out.reshape(orig_shape)
@dataclass
class MoEBlockFuser:
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
gate_name: str
scoring_func: str | None
"""`None` if routing stays in the Transformers gate."""
shared_name: str | None
shared_gate_name: str | None
router_dtype: torch.dtype | None = None
renormalize: bool | None = None
@staticmethod
def _match_router(
gate: nn.Module,
) -> tuple[str, torch.dtype | None, bool | None] | None:
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
`softmax`/`sigmoid`.
Returns the scoring function, the dtype the router computes in, and
whether the top-k weights are renormalized (`None` if the trace was cut
short before it could say)."""
state = {name for name, _ in named_state(gate)}
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
return None
graph = trace(gate)
if graph is None:
return None
# The routing top-k is the last one; any earlier one scores expert groups.
topks = [node for node in graph.nodes if is_op(node, "topk")]
if not topks:
return None
topk = topks[-1]
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
post_topk = False
# Find all scoring functions that reach the top-k node.
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
if not scorers:
# Find all scoring functions that are users of the top-k node.
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
post_topk = True
if len(scorers) != 1:
return None
scorer = scorers[0]
logits_cone = _reaches(scorer, "all_input_nodes")
if not any(is_op(n, "linear") for n in logits_cone):
return None
is_softmax = is_op(scorer, "softmax")
# vLLM always scores every expert, so matching a router that softmaxes
# the selection alone takes renormalizing. Otherwise look for the
# division, reading its absence as "no" only from a complete trace.
renormalize: bool | None
if (post_topk and is_softmax) or _renormalizes(topk):
renormalize = True
else:
renormalize = False if is_complete(graph) else None
scoring_func = "softmax" if is_softmax else "sigmoid"
return scoring_func, _forced_dtype(logits_cone), renormalize
@staticmethod
def _match_experts(
graph: fx.Graph, experts_name: str, moe_block: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
being the block's input.
Returns the experts call, `x` and the gate call."""
experts = find_node(
graph, lambda n: n.op == "call_module" and n.target == experts_name
)
if experts is None:
return None
# Bind by signature so keyword calls match too. Fail closed on any other
# argument: the rewritten forward would drop it.
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
try:
bound = forward.bind(*experts.args, **experts.kwargs)
except TypeError:
return None
if len(bound.arguments) != 3:
return None
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
return None
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
return None
gate = ids.args[0]
if not (
_reads_input(gate, x, moe_block)
and ids.args == (gate, 2)
and weights.args == (gate, 1)
):
return None
return experts, x, gate
@staticmethod
def _match_shared_experts(
node: object, x: fx.Node, moe_block: nn.Module
) -> tuple[str, str | None] | None:
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
Returns the shared expert's name and its optional gate's name."""
if _reads_input(node, x, moe_block):
return node.target, None
if not (is_op(node, "mul") and len(node.args) == 2):
return None
for gated, shared in (node.args, node.args[::-1]):
gated, shared = peel(gated), peel(shared)
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
continue
gate = peel(gated.args[0])
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
moe_block.get_submodule(gate.target)
):
return shared.target, gate.target
return None
@classmethod
def match(
cls,
moe_block: nn.Module,
experts_name: str,
divides_shared_output: bool = False,
) -> "MoEBlockFuser | None":
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
reshapes, casts and `nn.Identity`.
Args:
moe_block: The MoE block.
experts_name: The name of the experts in `moe_block`.
divides_shared_output: Whether vLLM routing would divide the shared
expert's output by `routed_scaling_factor` (fp16 overflow
protection), which Transformers decoder layers do not undo.
Returns:
A fuser if more than the experts can run in vLLM, else `None`.
"""
# Standard MoE block returns a single tensor.
if _returns_tuple(type(moe_block)):
return None
if (graph := trace(moe_block)) is None:
return None
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
return None
experts, x, gate = matched
unwrap = functools.partial(_unwrap, moe_block=moe_block)
gate_name = gate.target
router = cls._match_router(getattr(moe_block, gate_name))
scoring_func, router_dtype, renormalize = router or (None, None, None)
def is_routed(n: object) -> bool:
return n is experts or (
is_op(n, "mul")
and not isinstance(n.args[1], fx.Node)
and unwrap(n.args[0]) is experts
)
# Output: the experts' output, plus the shared expert's if any.
routed = out = unwrap(output_value(graph))
shared = None
if not is_routed(out):
if not (is_op(out, "add") and len(out.args) == 2):
return None
routed, shared = map(unwrap, out.args)
if is_routed(shared):
routed, shared = shared, routed
if not is_routed(routed):
return None
shared_name = shared_gate_name = None
if shared is not None:
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
return None
shared_name, shared_gate_name = match
# Fail closed: `rewrite_forward` runs only the experts and the detected
# shared expert, so any other stateful child would be dropped.
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
for name, child in moe_block.named_children():
if name not in accounted and next(named_state(child), None) is not None:
return None
if scoring_func is not None and shared_name and divides_shared_output:
logger.warning_once(
"%s could be fused but routing it in vLLM would apply "
"`routed_scaling_factor` by dividing the shared expert output, "
"which only fp16 overflow protection expects the decoder layer to "
"compensate for. Falling back to routing in Transformers; run in "
"bfloat16 to fuse it.",
type(moe_block).__name__,
)
scoring_func = router_dtype = renormalize = None
# Without vLLM routing, only a shared expert can move to vLLM, and only
# if the block does not scale the experts' output (vLLM routing does).
if scoring_func is None and (shared_name is None or routed is not experts):
return None
return cls(
gate_name,
scoring_func,
shared_name,
shared_gate_name,
router_dtype,
renormalize,
)
def gate(
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
) -> GateLinear:
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
hf_gate = getattr(moe_block, self.gate_name)
num_experts, hidden_size = hf_gate.weight.shape
gate = GateLinear(
hidden_size,
num_experts,
bias=False,
out_dtype=out_dtype or self.router_dtype,
prefix=maybe_prefix(prefix, self.gate_name),
)
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
gate.register_buffer("e_score_correction_bias", bias)
setattr(moe_block, self.gate_name, gate)
return gate
def shared_experts(
self, moe_block: nn.Module, prefix: str
) -> SharedExpertMLP | None:
"""Build the HF shared expert (and its optional gate)
as a `SharedExpertMLP` for vLLM's fused MoE."""
if self.shared_name is None:
return None
shared_experts = getattr(moe_block, self.shared_name)
gate = None
if self.shared_gate_name is not None:
hf_gate = getattr(moe_block, self.shared_gate_name)
gate = ReplicatedLinear(
hf_gate.in_features,
hf_gate.out_features,
bias=hf_gate.bias is not None,
prefix=maybe_prefix(prefix, self.shared_gate_name),
)
setattr(moe_block, self.shared_gate_name, gate)
return SharedExpertMLP(shared_experts, gate)
def rewrite_forward(self, moe_block: nn.Module) -> None:
"""Rewrite `moe_block.forward` to route through vLLM's fused MoE."""
moe_block.forward = types.MethodType(_moe_block_forward, moe_block)
@@ -1,164 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
RewriteFuser,
fused_head_size,
local_output_sizes,
)
from vllm.model_executor.models.transformers.fx_utils import (
compile_forward,
is_method,
recover_forward,
returned_linear,
upstream_linear,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class PackedQKVFuser(RewriteFuser):
"""Fuser for attention with q, k and v packed into one projection."""
qkv_name: str
o_name: str | None
q_size: int
kv_size: int
def info(self, name: str) -> str:
return (
f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear"
)
@staticmethod
def _packed_sizes(node: fx.Node) -> tuple[int, int] | None:
"""`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one."""
if not is_method(node, "split") or len(node.args) < 2:
return None
sizes = node.args[1]
if not isinstance(sizes, (tuple, list)) or len(sizes) != 3:
return None
if not all(isinstance(size, int) for size in sizes):
return None
q_size, k_size, v_size = sizes
if k_size != v_size or q_size < k_size:
return None
return q_size, k_size
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None":
for node in graph.nodes:
if (sizes := cls._packed_sizes(node)) is None:
continue
q_size, kv_size = sizes
qkv_node = upstream_linear(node.args[0], module)
if qkv_node is None:
continue
qkv_name = str(qkv_node.target)
# The split must consume the whole projection.
if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size:
continue
# o_proj produces the module's output and consumes the query width.
o_name = returned_linear(graph, module)
if o_name == qkv_name or (
o_name is not None
and module.get_submodule(o_name).in_features != q_size
):
o_name = None
return cls(
source_cls=type(module).__name__,
qkv_name=qkv_name,
o_name=o_name,
q_size=q_size,
kv_size=kv_size,
)
return None
def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call:
"""The unique `self.<qkv_name>(...)....split((a, b, c), ...)` call."""
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr == "split"
and node.args
and isinstance(node.args[0], (ast.Tuple, ast.List))
and len(node.args[0].elts) == 3
and any(
isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name
for inner in ast.walk(node.func.value)
)
]
if len(calls) != 1:
raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits")
return calls[0]
def update_forward(self, module: nn.Module) -> None:
"""Rewrite the split sizes to the sharded projection's per-rank widths."""
funcdef, fn = recover_forward(type(module))
split = self._split_call(funcdef)
# (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes]
sections = local_output_sizes(self.qkv_name)
split.args[0] = ast.parse(sections, mode="eval").body
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible with a head-sharded packed GEMM."""
head_size = fused_head_size(module, vllm_config)
qkv = module.get_submodule(self.qkv_name)
compatible = (
self.q_size % head_size == 0
and self.kv_size % head_size == 0
and qkv.out_features == self.q_size + 2 * self.kv_size
)
if not compatible:
logger.debug("%s is not compatible with packed QKV fusion", type(module))
return compatible
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = fused_head_size(module, vllm_config)
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
qkv = module.get_submodule(self.qkv_name)
merged = QKVParallelLinear(
hidden_size=qkv.in_features,
head_size=head_size,
total_num_heads=self.q_size // head_size,
total_num_kv_heads=self.kv_size // head_size,
bias=qkv.bias is not None,
quant_config=quant_config,
prefix=qkv_prefix,
return_bias=False,
)
setattr(module, self.qkv_name, merged)
log_replacement(qkv_prefix, qkv, merged)
# If there is an output projection, we know it must be rowwise.
if self.o_name is not None:
o_prefix = maybe_prefix(prefix, self.o_name)
o_proj = module.get_submodule(self.o_name)
new_o = replace_linear_class(
o_proj, "rowwise", quant_config, prefix=o_prefix
)
setattr(module, self.o_name, new_o)
log_replacement(o_prefix, o_proj, new_o)
@@ -1,209 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""QKV projection fuser: `q(x), k(x), v(x)` -> a fused qkv linear + split."""
import ast
from dataclasses import dataclass
from typing import TYPE_CHECKING, ClassVar
from torch import fx, nn
from vllm.logger import init_logger
from vllm.model_executor.layers.linear import QKVParallelLinear
from vllm.model_executor.models.transformers.fusers.base import (
StackedFuser,
fused_head_size,
)
from vllm.model_executor.models.transformers.fx_utils import (
block_chain,
compile_forward,
is_linear,
recover_forward,
returned_linear,
)
from vllm.model_executor.models.transformers.utils import (
log_replacement,
replace_linear_class,
)
from vllm.model_executor.models.utils import ShardId, maybe_prefix
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class QKVFuser(StackedFuser):
"""Fuser for the attention QKV pattern `q(x), k(x), v(x)`."""
q_name: str
k_name: str
v_name: str
o_name: str | None
merged_name: ClassVar[str] = "qkv_proj"
merged_cls_name: ClassVar[str] = "QKVParallelLinear"
@property
def shards(self) -> list[tuple[str, ShardId]]:
return [(self.q_name, "q"), (self.k_name, "k"), (self.v_name, "v")]
@classmethod
def _get_qkv_nodes(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
"""Search `graph` for the QKV pattern `q(x), k(x), v(x)`."""
by_input: dict[fx.Node, list[fx.Node]] = {}
for node in graph.nodes:
if (
is_linear(node, module)
and len(node.args) == 1
and not node.kwargs
and isinstance(node.args[0], fx.Node)
and node.args[0].op == "placeholder"
):
by_input.setdefault(node.args[0], []).append(node)
triples = [nodes for nodes in by_input.values() if len(nodes) == 3]
if len(triples) != 1:
return None
q_node, k_node, v_node = nodes = triples[0]
outs = [module.get_submodule(node.target).out_features for node in nodes]
if len(set(outs)) == 2:
# q is identified as the larger projection (GQA)
(q_node,) = (n for n, out in zip(nodes, outs) if outs.count(out) == 1)
k_node, v_node = (n for n, out in zip(nodes, outs) if outs.count(out) == 2)
if module.get_submodule(q_node.target).out_features != max(outs):
return None
elif len(set(outs)) != 1:
return None
return q_node, k_node, v_node
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "QKVFuser | None":
if (qkv_nodes := cls._get_qkv_nodes(graph, module)) is None:
return None
q, k, v = qkv_nodes
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
# o_proj produces the module's output.
o_name = returned_linear(graph, module)
# o_proj must be compatible with the q/k/v projections.
if o_name in names.values() or (
o_name is not None
and module.get_submodule(o_name).in_features
!= module.get_submodule(q.target).out_features
):
o_name = None
names["o_name"] = o_name
return cls(source_cls=type(module).__name__, **names)
def update_forward(self, module: nn.Module) -> None:
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source.
A projection may be guarded by an existence check -- a
`self.<proj> is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare
truthiness test -- which is folded to its constant value (see
`bypass_existence_guard`). The calls may sit in different branches, so the
fused GEMM is inserted before the earliest of them, in the innermost block
that dominates all three.
"""
funcdef, fn = recover_forward(type(module))
proj_names = (self.q_name, self.k_name, self.v_name)
calls = self._unguarded_calls(funcdef, proj_names)
arg_dumps = {ast.dump(call.args[0]) for call in calls}
if len(arg_dumps) != 1:
raise ValueError("projection inputs are written differently")
# The trace may be partial, so prove projection exclusivity in source:
# no other linear child may consume the same input (else the matched
# three may not be q, k and v)
other_linears = {
name
for name, child in module.named_children()
if isinstance(child, nn.Linear)
} - set(proj_names)
for node in ast.walk(funcdef):
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr in other_linears
and any(ast.dump(arg) in arg_dumps for arg in node.args)
):
raise ValueError("another linear consumes the same input")
# Insert the fused GEMM before the earliest call, in the innermost block
# common to all three (the calls may be split across branches).
chains = [block_chain(funcdef.body, call) for call in calls]
if any(not chain for chain in chains):
raise ValueError("projection calls not found in the function body")
depth = 0
for level in zip(*chains):
if len({id(block) for block, _ in level}) != 1:
break
depth += 1
block = chains[0][depth - 1][0]
indices = [chain[depth - 1][1] for chain in chains]
insert_index = min(indices)
self._check_input_stable(funcdef, module, calls, block, indices)
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
self._splice_merged_split(funcdef, calls, block, insert_index)
self.fused_forward = compile_forward(funcdef, fn)
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
v = module.get_submodule(self.v_name)
head_size = fused_head_size(module, vllm_config)
compatible = (
q.in_features == k.in_features == v.in_features
and len({proj.bias is None for proj in (q, k, v)}) == 1
and k.out_features == v.out_features
and q.out_features % head_size == 0
and k.out_features % head_size == 0
)
if not compatible:
logger.debug("%s is not compatible with QKV fusion", type(module))
return compatible
def update_attrs(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> None:
quant_config = vllm_config.quant_config
head_size = fused_head_size(module, vllm_config)
q = module.get_submodule(self.q_name)
k = module.get_submodule(self.k_name)
merged = QKVParallelLinear(
hidden_size=q.in_features,
head_size=head_size,
total_num_heads=q.out_features // head_size,
total_num_kv_heads=k.out_features // head_size,
bias=q.bias is not None,
quant_config=quant_config,
prefix=maybe_prefix(prefix, self.merged_name),
return_bias=False,
)
logger.debug(
"%s: %s, %s: %s, %s: %s -> %s: %s",
self.q_name,
q,
self.k_name,
k,
self.v_name,
module.get_submodule(self.v_name),
self.merged_name,
merged,
)
setattr(module, self.merged_name, merged)
# Drop the consumed submodules so their (meta) params are not expected.
for name in (self.q_name, self.k_name, self.v_name):
delattr(module, name)
# If there is an output projection, we know it must be rowwise.
if self.o_name is not None:
o_proj_prefix = maybe_prefix(prefix, self.o_name)
o_proj = module.get_submodule(self.o_name)
new_o = replace_linear_class(
o_proj, "rowwise", quant_config, prefix=o_proj_prefix
)
setattr(module, self.o_name, new_o)
log_replacement(o_proj_prefix, o_proj, new_o)
@@ -1,297 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""RMSNorm fuser: detect the norm structurally and swap in vLLM's fused RMSNorm."""
from dataclasses import dataclass
from typing import TYPE_CHECKING
import torch
from torch import fx, nn
from vllm.distributed import (
get_tensor_model_parallel_rank,
get_tensor_model_parallel_world_size,
tensor_model_parallel_all_gather,
)
from vllm.distributed.parallel_state import model_parallel_is_initialized
from vllm.distributed.utils import split_tensor_along_last_dim
from vllm.logger import init_logger
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
from vllm.model_executor.models.transformers.fx_utils import (
find_node,
forward_input_count,
is_op,
output_value,
peel,
trace,
)
from vllm.model_executor.models.transformers.layers import (
GemmaRMSNorm,
RMSNorm,
)
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
def _operand(node: fx.Node, index: int, name: str) -> object | None:
"""Operand `index` of `node`, whether it was passed positionally or as `name`."""
if len(node.args) > index:
return node.args[index]
return node.kwargs.get(name)
def _is_squared(node: object, x: fx.Node) -> bool:
"""`x**2`, `x.square()` or `x * x`, through any dtype casts."""
node = peel(node)
if is_op(node, "pow"):
return (
peel(_operand(node, 0, "input")) is x and _operand(node, 1, "exponent") == 2
)
if is_op(node, "square"):
return peel(_operand(node, 0, "input")) is x
if is_op(node, "mul"):
return (
peel(_operand(node, 0, "input")) is x
and peel(_operand(node, 1, "other")) is x
)
return False
def _is_inverse_sqrt(node: object) -> bool:
"""Detect `rsqrt(v)`, or the `pow(v, -0.5)` / `v ** -0.5` spelling of it."""
if is_op(node, "rsqrt"):
return True
if is_op(node, "pow"):
return _operand(node, 1, "exponent") == -0.5
return False
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
"""`eps` from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
add = peel(_operand(rsqrt, 0, "input"))
if not is_op(add, "add"):
return None
operands = [_operand(add, 0, "input"), _operand(add, 1, "other")]
consts = [a for a in operands if isinstance(a, (int, float))]
nodes = [a for a in operands if isinstance(a, fx.Node)]
if len(consts) != 1 or len(nodes) != 1:
return None
mean = peel(nodes[0])
if not is_op(mean, "mean"):
return None
if not _is_squared(_operand(mean, 0, "input"), x):
return None
return float(consts[0])
def _is_one_plus(node: object) -> bool:
"""`1 + weight` in either operand order (marks a zero-centered weight)."""
node = peel(node)
if not is_op(node, "add"):
return False
return any(
isinstance(a, (int, float)) and a == 1
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
)
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
"""Does the forward compute anything after `node` before returning?"""
value = output_value(graph)
return value is not None and peel(value) is not node
class TPAwareNormMixin(nn.Module):
"""Mixin for RMSNorms that reconstructs a TP-sharded input before normalizing."""
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if model_parallel_is_initialized():
self.tp_size = get_tensor_model_parallel_world_size()
self.tp_rank = get_tensor_model_parallel_rank()
else:
self.tp_size, self.tp_rank = 1, 0
def forward(
self, x: torch.Tensor, residual: torch.Tensor | None = None
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
if self.tp_size > 1 and x.shape[-1] < (full := self.weight.shape[0]):
if x.shape[-1] * self.tp_size != full:
raise ValueError(
f"Cannot gather norm of width {full}: a TP-sharded input of "
f"width {x.shape[-1]} does not tile it evenly across "
f"{self.tp_size} ranks (replicated or uneven sharding)."
)
x = tensor_model_parallel_all_gather(x.contiguous())
x = super().forward(x)
splits = split_tensor_along_last_dim(x, num_partitions=self.tp_size)
return splits[self.tp_rank]
return super().forward(x, residual)
class TPAwareRMSNorm(TPAwareNormMixin, RMSNorm):
"""`RMSNorm` that reconstructs a TP-sharded input before normalizing."""
class TPAwareGemmaRMSNorm(TPAwareNormMixin, GemmaRMSNorm):
"""`GemmaRMSNorm` that reconstructs a TP-sharded input before normalizing."""
@dataclass
class RMSNormFuser(BaseFuser):
"""Fuser for RMSNorm patterns, including Gemma-style zero-centered weights."""
zero_centered: bool
"""Gemma-style `(1 + weight)` scaling (weight initialised at zero)."""
source_cls: str
"""Class name of the norm this was matched from (for logging)."""
eps_attr: str | None = None
"""Attribute holding `eps`, read per instance in `fuse`."""
eps: float | None = None
"""`eps` itself, when it is not held in an attribute (see `_eps_source`)."""
def info(self, name: str) -> str:
norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm"
return f"Fused: {name} ({self.source_cls}) -> {norm} (CustomOp)"
@classmethod
def match(cls, graph: fx.Graph, module: nn.Module) -> "RMSNormFuser | None":
"""Match a graph to the RMSNorm pattern, returning a fuser if found."""
if forward_input_count(type(module)) != 1:
return None
x = find_node(graph, lambda n: n.op == "placeholder")
if x is None:
return None
# Handle native torch `rms_norm` op.
rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm"))
if rms_norm is not None and peel(_operand(rms_norm, 0, "input")) is x:
if _has_trailing_compute(graph, rms_norm):
return None
eps_attr, eps = cls._eps_source(graph, module)
return cls(
zero_centered=False,
source_cls=type(module).__name__,
eps_attr=eps_attr,
eps=eps,
)
# Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern.
# The rsqrt over the mean-square variance is the spine of the norm.
rsqrt = None
for node in graph.nodes:
if _is_inverse_sqrt(node) and _variance_eps(node, x) is not None:
rsqrt = node
break
if rsqrt is None:
return None
# The `x * rsqrt(...)` normalize multiply.
normalize = find_node(
graph,
lambda n: is_op(n, "mul")
and rsqrt in map(peel, (_operand(n, 0, "input"), _operand(n, 1, "other"))),
)
if normalize is None:
return None
# An optional later `weight * normalized` (or `(1 + weight) * normalized`).
tail, zero_centered = normalize, False
for node in graph.nodes:
if not is_op(node, "mul") or node is normalize:
continue
operands = [
peel(a)
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
if isinstance(a, fx.Node)
]
if len(operands) == 2 and normalize in operands:
weight = next(o for o in operands if o is not normalize)
tail, zero_centered = node, _is_one_plus(weight)
break
# The norm must be the last compute in forward, or it is not a pure norm.
if _has_trailing_compute(graph, tail):
return None
eps_attr, eps = cls._eps_source(graph, module)
return cls(
zero_centered=zero_centered,
source_cls=type(module).__name__,
eps_attr=eps_attr,
eps=eps,
)
@classmethod
def _eps_source(
cls, graph: fx.Graph, module: nn.Module
) -> tuple[str | None, float | None]:
"""Where `fuse` should read `eps` from, resolved once per class."""
eps = cls._eps_from_graph(graph)
if eps is None:
return None, None
# Whatever supplied the constant must still equal it.
candidates = {
name: value
for name, value in vars(module).items()
if isinstance(value, float) and value == eps
}
# Use unique markers and retrace to verify exactly which attribute is eps.
markers = {float(-index - 1): name for index, name in enumerate(candidates)}
marked = None
if markers:
try:
for marker, name in markers.items():
setattr(module, name, marker)
if (remarked := trace(module)) is not None:
marked = cls._eps_from_graph(remarked)
finally:
for name, value in candidates.items():
setattr(module, name, value)
if marked is not None and (name := markers.get(marked)) is not None:
return name, None
logger.debug_once(
"%s does not hold its eps (%s) in an attribute. Every instance in this "
"model will use the value traced from this instance. If this is not "
"desired, consider storing and reading eps using attribute of %s.",
type(module).__name__,
eps,
type(module).__name__,
)
return None, eps
@staticmethod
def _eps_from_graph(graph: fx.Graph) -> float | None:
"""Extract the `eps` constant from the graph, if present."""
if (x := find_node(graph, lambda n: n.op == "placeholder")) is None:
return None
fused = find_node(graph, lambda n: is_op(n, "rms_norm"))
if fused is not None and peel(_operand(fused, 0, "input")) is x:
args, kwargs = fused.args, fused.kwargs
eps = args[3] if len(args) > 3 else kwargs.get("eps")
return float(eps) if isinstance(eps, (int, float)) else None
for node in graph.nodes:
if _is_inverse_sqrt(node) and (eps := _variance_eps(node, x)) is not None:
return eps
return None
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
return True
def fuse(
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
) -> nn.Module:
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
weight = getattr(module, "weight", None)
hidden_size = weight.size(0) if weight is not None else 0
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
if not isinstance(eps, (int, float)):
# If eps was not detected, match torch behaviour.
config_dtype = vllm_config.model_config.dtype
dtype = weight.dtype if weight is not None else config_dtype
eps = torch.finfo(dtype).eps
if self.zero_centered:
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
return TPAwareRMSNorm(
hidden_size=hidden_size,
eps=eps,
has_weight=weight is not None,
dtype=weight.dtype if weight is not None else None,
)
@@ -1,818 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""fx tracing and forward-source rewriting for the Transformers backend fusers.
A small engine, independent of any particular pattern: trace a module's forward
with `torch.fx` (tolerating a partial graph), inspect the resulting nodes, and
rewrite the forward's *source* (AST) so only matched calls change while the rest
stays live Python. `fusion.py` builds the concrete fusion patterns on top.
"""
import ast
import contextlib
import inspect
import operator
import textwrap
from collections.abc import Callable
from itertools import chain
from typing import Any
from unittest import mock
import torch
from torch import fx, nn
from torch.nn import functional as F
from typing_extensions import TypeIs
from vllm.logger import init_logger
logger = init_logger(__name__)
_UNKNOWN: Any = object()
"""Sentinel meta value for proxies whose concrete value could not be inferred.
Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`)."""
_MODULE_CALL = nn.Module.__call__
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
`call_module` nodes; meta execution must call modules for real."""
_MODULE_GETATTR = nn.Module.__getattr__
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
recurse until the stack runs out, truncating the graph."""
@contextlib.contextmanager
def _unpatched_modules():
"""Restores the module hooks fx patches, so meta execution records nothing."""
with (
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
):
yield
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
return isinstance(node, fx.Node) and node.meta.get("leaf_call", False)
def _reference_weight(module: nn.Module) -> torch.Tensor | None:
"""A weight whose trailing dim is the module's hidden size.
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
for child in module.modules():
if isinstance(child, nn.Linear):
return child.weight
for param in module.parameters():
if param.ndim in (1, 2):
return param
return None
class _MetaProxy(fx.Proxy):
"""Proxy carrying the meta-tensor value of the traced expression.
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
executing each op on the meta values, so PyTorch's meta kernels are the
single source of shape inference — no per-op rules."""
meta: Any = _UNKNOWN
def __len__(self) -> int:
if self.meta is not _UNKNOWN:
return len(self.meta)
return super().__len__() # type: ignore[misc]
def __getattr__(self, k: str) -> "_MetaAttribute":
return _MetaAttribute(self, k)
def __setitem__(self, key: object, value: object) -> None:
# Record `obj[key] = value` so tracing continues past in-place writes
# into containers (e.g. Gemma 4's `shared_kv_states[...] = ...`).
self.tracer.create_proxy(
"call_function", operator.setitem, (self, key, value), {}
)
class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
`Tracer.proxy`, so the meta value must be grafted on here too."""
def __init__(self, root: fx.Proxy, attr: str):
super().__init__(root, attr)
root_meta = getattr(root, "meta", _UNKNOWN)
if root_meta is not _UNKNOWN:
with contextlib.suppress(Exception):
self.meta = getattr(root_meta, attr)
class _AllLeafTracer(fx.Tracer):
"""Tracer that treats every submodule as a leaf.
Each child stays one `call_module` node, so matching sees the module's own
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
Every traced op is also executed on meta tensors (see `_MetaProxy`) so
shape unpacks and `*`-splats trace through; anything else untraceable ends
the trace early and the partial graph is matched.
"""
varkw: str | None = None
"""Name of the traced forward's `**kwargs` parameter, if any."""
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
return True
def proxy(self, node: fx.Node) -> fx.Proxy:
return _MetaProxy(node, self)
def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs):
proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs)
if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN:
# A failure stays _UNKNOWN; only fatal if a shape question is asked.
with contextlib.suppress(Exception):
proxy.meta = self._infer_meta(kind, target, args, kwargs)
return proxy
def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict):
"""Execute the op on meta tensors; PyTorch infers the output value."""
if kind == "placeholder":
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states.
weight = _reference_weight(self.root)
if str(target) == "hidden_states" and weight is not None:
return torch.empty(
1, 8, weight.shape[-1], dtype=weight.dtype, device="meta"
)
return _UNKNOWN
if kind == "get_attr":
with _unpatched_modules():
value = operator.attrgetter(str(target))(self.root)
if isinstance(value, torch.Tensor):
value = torch.empty_like(value, device="meta")
return value
unknown = False
def meta_of(arg: object) -> object:
nonlocal unknown
if isinstance(arg, fx.Proxy):
meta = getattr(arg, "meta", _UNKNOWN)
unknown = unknown or meta is _UNKNOWN
return meta
return arg
meta_args = fx.node.map_aggregate(args, meta_of)
meta_kwargs = fx.node.map_aggregate(kwargs, meta_of)
if unknown:
return _UNKNOWN
if kind == "call_function":
assert callable(target)
return target(*meta_args, **meta_kwargs)
if kind == "call_method":
receiver, *rest = meta_args
return getattr(receiver, str(target))(*rest, **meta_kwargs)
if kind == "call_module":
# Run the child's forward with all its state on "meta", without
# mutating it (at match time params may be meta but buffers real).
child = self.root.get_submodule(str(target))
state = {
name: torch.empty_like(tensor, device="meta")
for name, tensor in chain(
child.named_parameters(), child.named_buffers()
)
}
with _unpatched_modules():
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
return _UNKNOWN
def _is_varkw(self, node: object) -> bool:
return (
isinstance(node, fx.Node)
and node.op == "placeholder"
and str(node.target).lstrip("*") == self.varkw
)
def iter(self, obj: fx.Proxy):
# Assume kwargs is always empty to simplify tracing.
node = obj.node
if self._is_varkw(node) or (
node.op == "call_method"
and node.target == "keys"
and self._is_varkw(node.args[0])
):
return iter(())
meta = getattr(obj, "meta", _UNKNOWN)
if meta is _UNKNOWN:
return super().iter(obj)
return iter([obj[i] for i in range(len(meta))])
def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable:
"""Wrap any callable so tracing records it as one opaque `call_function` node.
Lets the trace continue past untraceable bodies. Only the proxy arguments carry into
the node's dataflow; the rest are dropped rather than lifted into the graph.
`length` declares how many values the callable returns, so unpacking its result also
traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough.
"""
def leaf(*args, **kwargs):
proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy))
if not proxies:
return fn(*args, **kwargs)
proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {})
proxy.node.meta["leaf_call"] = True
if length is not None:
# The body never executes, so fabricate a value of the declared length.
proxy.meta = (_UNKNOWN,) * length
return proxy
return leaf
def _leaf_attention_interfaces():
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
`vllm_attention_function` needs runtime context so it is untraceable.
Every interface returns `(attn_output, attn_weights)`."""
from transformers.modeling_utils import AttentionInterface
original = AttentionInterface.get_interface
def get_interface(self, *args, **kwargs):
return _as_leaf_call(original(self, *args, **kwargs), length=2)
return mock.patch.object(AttentionInterface, "get_interface", get_interface)
def trace(module: nn.Module) -> fx.Graph | None:
"""Trace `module.forward`, returning the partial graph on failure."""
parameters = forward_parameters(type(module))
# vLLM never passes `past_key_values` so it is always the default value of `None`.
# Make this concrete to simplify tracing.
concrete_args = None
if "past_key_values" in parameters:
concrete_args = {"past_key_values": None}
# Get the name of the kwargs parameter passed to module.forward (usually "kwargs")
tracer = _AllLeafTracer()
tracer.varkw = next(
(
p.name
for p in parameters.values()
if p.kind is inspect.Parameter.VAR_KEYWORD
),
None,
)
try:
with _leaf_attention_interfaces():
return tracer.trace(module, concrete_args=concrete_args)
except Exception as exc:
logger.debug("Could not fully trace %s: %s", type(module), exc)
return getattr(tracer, "graph", None)
def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
"""Parse the source of `cls.forward`, ready for rewriting."""
fn = inspect.unwrap(cls.forward)
if fn.__code__.co_freevars:
raise ValueError("forward is a closure")
tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
funcdef = tree.body[0]
if not isinstance(funcdef, ast.FunctionDef):
raise ValueError("source is not a plain function definition")
# `fn` is already unwrapped; don't re-apply its decorators
funcdef.decorator_list.clear()
# Annotations may not evaluate outside the defining module (e.g. with
# postponed evaluation); they're not needed at runtime
funcdef.returns = None
args = funcdef.args
for arg in (
*args.posonlyargs,
*args.args,
*args.kwonlyargs,
*filter(None, (args.vararg, args.kwarg)),
):
arg.annotation = None
# Recompiling outside the class body would break name mangling
for node in ast.walk(funcdef):
name = getattr(node, "attr", None) or getattr(node, "id", None)
if name and name.startswith("__") and not name.endswith("__"):
raise ValueError(f"{name} would be name mangled")
return funcdef, fn
def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
"""`cls.forward`'s signature parameters, or empty if uninspectable."""
try:
return dict(inspect.signature(cls.forward).parameters)
except (TypeError, ValueError):
return {}
def forward_input_count(cls: type[nn.Module]) -> int:
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
any `*args`/`**kwargs`. Read from the signature, so it is independent of
whether the trace completes (unlike counting placeholders)."""
params = list(forward_parameters(cls).values())
if not params:
return 1 # uninspectable: assume a single input and let matching decide
fixed = (
inspect.Parameter.POSITIONAL_ONLY,
inspect.Parameter.POSITIONAL_OR_KEYWORD,
inspect.Parameter.KEYWORD_ONLY,
)
return sum(1 for p in params[1:] if p.kind in fixed)
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
"""Compile `funcdef` in `fn`'s module so tracebacks point at the source."""
module = ast.Module(body=[funcdef], type_ignores=[])
ast.fix_missing_locations(module)
ast.increment_lineno(module, fn.__code__.co_firstlineno - 1)
code = compile(module, fn.__code__.co_filename, "exec")
namespace: dict = {}
exec(code, fn.__globals__, namespace)
return namespace[funcdef.name]
def self_call_and_refs(
funcdef: ast.FunctionDef, name: str
) -> tuple[ast.Call, list[ast.Attribute]]:
"""The unique `self.<name>(arg)` call, plus every other `self.<name>` reference.
Raises unless exactly one single-argument `self.<name>(arg)` call exists, so
one fx `call_module` node maps to one syntactic call site. The remaining
references (e.g. `is not None` guards) are returned for the caller to resolve.
"""
uses = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Attribute)
and node.attr == name
and isinstance(node.value, ast.Name)
and node.value.id == "self"
]
calls = [
node
for node in ast.walk(funcdef)
if isinstance(node, ast.Call)
and any(node.func is use for use in uses)
and len(node.args) == 1
and not isinstance(node.args[0], ast.Starred)
and not node.keywords
]
if len(calls) != 1:
raise ValueError(f"{name} is not called exactly once as self.{name}(arg)")
call = calls[0]
other_refs = [use for use in uses if use is not call.func]
return call, other_refs
def single_self_call(funcdef: ast.FunctionDef, name: str) -> ast.Call:
"""The unique `self.<name>(arg)` call in `funcdef`.
Raises unless `name` appears exactly once, as such a call, so the source
rewrite agrees with the fx match.
"""
call, other_refs = self_call_and_refs(funcdef, name)
if other_refs:
raise ValueError(f"{name} is referenced {len(other_refs) + 1} times")
return call
def _is_none(node: ast.expr) -> bool:
return isinstance(node, ast.Constant) and node.value is None
def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
"""Is `ref` used only for its truth value (a test, or a `not` operand)?
In these positions the object's identity never escapes, so a reference that
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
yield an operand, so the module could escape (`x and self.<name>`)."""
for node in ast.walk(funcdef):
if (
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
and node.test is ref
):
return True
if (
isinstance(node, ast.UnaryOp)
and isinstance(node.op, ast.Not)
and node.operand is ref
):
return True
return False
def bypass_existence_guard(
funcdef: ast.FunctionDef, ref: ast.Attribute, name: str
) -> None:
"""Fold a guard on `self.<name>`'s existence to its constant value.
The fuser deletes `self.<name>` and binds only instances where it exists as a
truthy `nn.Linear` (guaranteed by the match, the cache key, and `validate`),
so a guard testing its presence is a fusion invariant. Two forms are folded:
- an identity `None` check `self.<name> is None` -> `False`,
`self.<name> is not None` -> `True` (either operand order). `==`/`!=` are
*not* folded: `is_linear` accepts `nn.Linear` subclasses, which may override
`__eq__`/`__ne__`, so equality is not guaranteed to track identity.
- a bare truthiness test (`if self.<name>:`, `... if self.<name> else ...`,
`not self.<name>`): the reference itself to `True`.
Any other surviving reference escapes the projection's value, which no longer
exists after fusion, so refuse rather than change semantics."""
for node in ast.walk(funcdef):
# `self.<name> is (not) None`, either operand order.
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
continue
(op,), (right,) = node.ops, node.comparators
if not isinstance(op, (ast.Is, ast.IsNot)):
continue
if (ref is node.left and _is_none(right)) or (
ref is right and _is_none(node.left)
):
value = isinstance(op, ast.IsNot)
replace_expr(funcdef, node, ast.copy_location(ast.Constant(value), node))
return
# A bare truthiness test (statement `if`, ternary, or `not`); an `nn.Linear`
# is always truthy, so the reference folds to `True`.
if _in_boolean_context(funcdef, ref):
replace_expr(funcdef, ref, ast.copy_location(ast.Constant(True), ref))
return
raise ValueError(f"{name} is referenced outside an existence guard")
def block_chain(
block: list[ast.stmt], node: ast.AST
) -> list[tuple[list[ast.stmt], int]]:
"""Path of (statement list, index) pairs from `block` down to `node`.
Each pair names a nested block and the index in it of the statement
containing `node`; the last pair is `node`'s innermost block. Empty if
`node` is not in `block`. The longest common prefix of several nodes' chains
is the innermost block that dominates them all.
"""
for index, stmt in enumerate(block):
if not any(child is node for child in ast.walk(stmt)):
continue
child_blocks = [
getattr(stmt, fld, None) for fld in ("body", "orelse", "finalbody")
]
child_blocks += [h.body for h in getattr(stmt, "handlers", [])]
child_blocks += [c.body for c in getattr(stmt, "cases", [])]
for child_block in child_blocks:
if (
isinstance(child_block, list)
and child_block
and (tail := block_chain(child_block, node))
):
return [(block, index), *tail]
return [(block, index)]
return []
def _base_name(node: ast.expr) -> str | None:
"""The root `Name` of an attribute/subscript chain (`a.b[c]` -> `a`)."""
while isinstance(node, (ast.Attribute, ast.Subscript)):
node = node.value
return node.id if isinstance(node, ast.Name) else None
def _rebound_names(region: list[ast.stmt]) -> set[str]:
"""Names rebound outright within `region` (`x = ...`, `del x`).
These are `Name` nodes in a `Store`/`Del` context."""
return {
node.id
for stmt in region
for node in ast.walk(stmt)
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del))
}
def _inplace_target(node: ast.AST) -> str | None:
"""Base name `node` mutates in place, if any.
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
plain `Name` store (see `_rebound_names`)."""
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
node.ctx, (ast.Store, ast.Del)
):
return _base_name(node)
if (
isinstance(node, ast.Call)
and isinstance(node.func, ast.Attribute)
and node.func.attr.endswith("_")
and not node.func.attr.endswith("__")
):
return _base_name(node.func.value)
return None
def _mutated_names(region: list[ast.stmt]) -> set[str]:
"""Names mutated in place within `region` (see `_inplace_target`)."""
return {
base
for stmt in region
for node in ast.walk(stmt)
if (base := _inplace_target(node)) is not None
}
def _call_name(call: ast.Call) -> str:
"""The called function's own name, without its qualifier."""
if isinstance(call.func, ast.Attribute):
return call.func.attr
return call.func.id if isinstance(call.func, ast.Name) else ""
def _arg_names(call: ast.Call) -> set[str]:
"""Every name `call` reads through its arguments."""
reads = [*call.args, *(kw.value for kw in call.keywords)]
return {
node.id
for arg in reads
for node in ast.walk(arg)
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
}
def _writes_through_args(call: ast.Call) -> bool:
"""Does `call` write through an argument rather than return a new value?
Torch spells this three ways: an `out=` destination, `inplace=True`, and the
trailing-underscore convention (`torch.relu_(x)`). The underscore is checked
on the function's own name, so it catches the free-function form that
`_inplace_target` cannot (there the base name is the module, `torch`).
"""
for keyword in call.keywords:
if keyword.arg == "out":
return True
if keyword.arg == "inplace" and not (
isinstance(keyword.value, ast.Constant) and keyword.value.value is False
):
return True
name = _call_name(call)
return name.endswith("_") and not name.endswith("__")
def written_names(region: list[ast.stmt]) -> set[str]:
"""Names `region` may rebind or write through.
Beyond plain rebinding and writes through the name itself (`_rebound_names`,
`_mutated_names`), a call can write through an argument without naming it as
a target: `F.relu(x, inplace=True)`, `torch.add(x, y, out=x)`,
`torch.relu_(x)`. A statement-level call whose result is discarded is treated
the same way, since it can only be there for a side effect.
"""
written = _rebound_names(region) | _mutated_names(region)
for stmt in region:
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
written |= _arg_names(stmt.value)
for node in ast.walk(stmt):
if isinstance(node, ast.Call) and _writes_through_args(node):
written |= _arg_names(node)
return written
_METADATA_ATTRS = frozenset(
{"shape", "dtype", "device", "ndim", "size", "dim", "numel", "item"}
)
def _aliasing_reads(node: ast.expr) -> set[str]:
"""Names `node` reads in a way that could yield a view of them.
Metadata (`x.shape`, `x.size(0)`, `x.dtype`) is ints and tuples sharing no
storage with `x`, so reaching a name only through metadata cannot alias it.
Shape arithmetic is pervasive between projections, so propagating through it
would taint nearly every later name.
"""
names: set[str] = set()
stack: list[ast.AST] = [node]
while stack:
current = stack.pop()
if isinstance(current, ast.Attribute) and current.attr in _METADATA_ATTRS:
continue
if (
isinstance(current, ast.Call)
and isinstance(current.func, ast.Attribute)
and current.func.attr in _METADATA_ATTRS
):
stack.extend(current.args)
stack.extend(keyword.value for keyword in current.keywords)
continue
if isinstance(current, ast.Name) and isinstance(current.ctx, ast.Load):
names.add(current.id)
stack.extend(ast.iter_child_nodes(current))
return names
def _returns_fresh(node: ast.expr, module: nn.Module) -> bool:
"""Does evaluating `node` allocate, rather than return a view of its input?
A `nn.Linear` writes its output into new storage, so its result aliases
nothing that went in, and a chain of view ops on top of it aliases only that
fresh tensor. Walking the receiver chain down to a `self.<linear>(...)` call
therefore proves the value cannot alias the projection input. A literal
aliases nothing either, and a branch or sequence is fresh when every part is
-- which is how a guarded projection (`p(x) if p is not None else None`)
stays untracked. A call on an attribute that is missing or `None` cannot run
at all, so a branch selecting a different configuration is fresh too.
"""
if isinstance(node, ast.Constant):
return True
if isinstance(node, ast.IfExp):
if isinstance(node.test, ast.Constant):
# A folded existence guard leaves the untaken branch as dead code.
taken = node.body if node.test.value else node.orelse
return _returns_fresh(taken, module)
return _returns_fresh(node.body, module) and _returns_fresh(node.orelse, module)
if isinstance(node, (ast.Tuple, ast.List)):
return all(_returns_fresh(elt, module) for elt in node.elts)
while isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
func = node.func
if isinstance(func.value, ast.Name) and func.value.id == "self":
attribute = getattr(module, func.attr, None)
if attribute is None or isinstance(attribute, nn.Linear):
# A missing or `None` attribute cannot be called, so the branch
# selecting it never runs (`q_proj` on a q-LoRA layer).
return True
# Anything else can only return storage it was handed, so it is
# fresh exactly when its inputs are (`q_norm(q_proj(x))`).
return bool(node.args) and all(
_returns_fresh(arg, module) for arg in node.args
)
node = func.value
return False
def aliasing_names(
funcdef: ast.FunctionDef, seed: set[str], module: nn.Module
) -> set[str]:
"""`seed` plus every name that may alias one of them.
A view shares storage with its base (`h = x.view(-1)`), so mutating `h`
mutates `x`; tracking only `seed` would miss that. Any assignment reading a
tracked name therefore taints its targets, transitively, unless its value is
freshly allocated (`_returns_fresh`).
"""
names = set(seed)
assigns = [
(node, node.value)
for node in ast.walk(funcdef)
if isinstance(node, (ast.Assign, ast.AnnAssign))
and node.value is not None
and not _returns_fresh(node.value, module)
]
changed = True
while changed:
changed = False
for node, value in assigns:
if not _aliasing_reads(value) & names:
continue
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
stores = {
child.id
for target in targets
for child in ast.walk(target)
if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store)
}
if stores - names:
names |= stores
changed = True
return names
def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
"""Replace the expression `old` (by identity) with `new` within `module`."""
class _Replacer(ast.NodeTransformer):
def visit(self, node: ast.AST) -> ast.AST:
if node is old:
return new
return super().generic_visit(node)
_Replacer().visit(module)
def is_complete(graph: fx.Graph) -> bool:
"""Did the trace run to completion? A partial graph has no `output` node.
Only a complete graph can be read for the *absence* of an op; in a partial
one the tail is exactly what is missing."""
return any(node.op == "output" for node in graph.nodes)
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
"""The first node in `graph` matching `predicate`, or `None`."""
return next((n for n in graph.nodes if predicate(n)), None)
def output_value(graph: fx.Graph) -> object | None:
"""The value the graph's `output` node returns, if the trace reached one."""
output = find_node(graph, lambda n: n.op == "output")
if output is None or not output.args:
return None
return output.args[0]
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
"""Nearest linear producing `node`, walking back through splits/reshapes.
Non-linear submodules are transparent too (e.g. the dropout GPT-style
attentions apply after their output projection). Never walks through a leaf
call (e.g. an attention interface): its inputs are what attention consumes,
not what produced the value."""
stack = [node]
seen: set[fx.Node] = set()
while stack:
current = stack.pop()
if not isinstance(current, fx.Node) or current in seen:
continue
seen.add(current)
if is_linear(current, module):
return current
if current.op in (
"call_function",
"call_method",
"call_module",
) and not is_leaf_call(current):
stack.extend(current.args)
return None
def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
Never walks through a leaf call (e.g. an attention interface): what crosses
it is consumed by the attention computation, not projected."""
queue = list(node.users)
seen: set[fx.Node] = set()
while queue:
current = queue.pop(0)
if current in seen:
continue
seen.add(current)
if is_linear(current, module):
return current
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
queue.extend(current.users)
return None
def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None:
"""Name of the Linear producing the graph's (first) output value."""
value = output_value(graph)
if isinstance(value, (tuple, list)) and value:
value = value[0]
linear = upstream_linear(value, module)
return None if linear is None else str(linear.target)
def is_linear(node: fx.Node, module: nn.Module) -> bool:
"""Is node `nn.Linear.__call__()`."""
return node.op == "call_module" and isinstance(
module.get_submodule(node.target), nn.Linear
)
_DTYPE_CASTS = frozenset({"to", "float", "double", "half", "bfloat16", "type_as"})
def peel(node: object) -> object:
"""Strip dtype-cast wrappers (`.to(...)`, `.float()`, `.type_as(...)`)."""
while (
isinstance(node, fx.Node)
and node.op == "call_method"
and node.target in _DTYPE_CASTS
):
node = node.args[0]
return node
def is_fn(node: object, target: object) -> TypeIs[fx.Node]:
"""Is node `<target>()`."""
return (
isinstance(node, fx.Node)
and node.op == "call_function"
and node.target is target
)
def is_method(node: object, name: str) -> TypeIs[fx.Node]:
"""Is node `.<name>()`."""
return (
isinstance(node, fx.Node) and node.op == "call_method" and node.target == name
)
def is_op(node: object, name: str) -> TypeIs[fx.Node]:
"""Is node `<mod>.<name>()` for torch, F, operator, or Tensor."""
return any(
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
) or (hasattr(torch.Tensor, name) and is_method(node, name))
@@ -1,64 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Layer provider resolution for the Transformers modeling backend.
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
resolved source of every symbol is logged so it is clear which layers run
hw-agnostic and which fell back to vLLM.
"""
import importlib
import vllm.envs as envs
from vllm.logger import init_logger
logger = init_logger(__name__)
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
_VLLM_PKG = "vllm.model_executor.layers"
def _resolve(module: str, name: str):
"""Return `name` from the hw-agnostic `module` when enabled and available,
else from vLLM. Logs which source was used."""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
logger.info("Using hw-agnostic layer: %s", name)
return obj
except (ImportError, AttributeError):
logger.warning(
"hw-agnostic layer %s is not available; falling back to default", name
)
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
RMSNorm = _resolve("layernorm", "RMSNorm")
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
def get_act_and_mul_fn(act_fn_name: str):
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
Resolved per call because the op is name-parameterized: an activation with
no hw-agnostic equivalent falls back to vLLM individually.
"""
if envs.VLLM_USE_HW_AGNOSTIC:
try:
from vllm.model_executor.hw_agnostic.layers.activation import (
get_act_and_mul_fn as hw_fn,
)
fn = hw_fn(act_fn_name)
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
return fn
except (ImportError, KeyError):
logger.warning_once(
"hw-agnostic activation %s is not available; falling back to vLLM",
act_fn_name,
)
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
return vllm_fn(act_fn_name)
@@ -1,80 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixin for legacy models."""
from typing import TYPE_CHECKING
import torch
from vllm.model_executor.models.utils import WeightsMapper
from vllm.sequence import IntermediateTensors
from .base import Base
if TYPE_CHECKING:
from vllm.config import VllmConfig
class LegacyMixin(Base):
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
self.hf_to_vllm_mapper |= WeightsMapper(
# Drop unsupported/unwanted output embeddings layers.
orig_to_new_prefix={
"model.lm_head.": None,
"model.predictions.": None,
"model.qa_outputs.": None,
"model.embeddings_project.": None,
"model.discriminator_predictions.": None,
},
orig_to_new_substr={
# Some encoder models have the position_ids buffer in the checkpoint.
# vLLM always passes position_ids as an argument, so drop the buffer.
"position_ids": None,
# Some encoder models have the bias of the final classifier layer in
# the checkpoint. vLLM does not use this bias, so drop it.
"score.bias": None,
},
)
# roberta-like models an extra padding in positions.
# FIXME(Isotr0py): This is quite hacky for roberta edge case,
# we should find a better way to handle this.
self.is_roberta = "roberta" in self.text_config.model_type
self.padding_idx = self.text_config.pad_token_id
def forward(
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor | IntermediateTensors:
if self.is_roberta:
# RoBERTa positions start at padding_idx + 1.
# Non-in-place add to avoid mutating the persistent GPU buffer --
# in-place += would accumulate on CUDA graph padding slots.
positions = positions + self.padding_idx + 1
return super().forward(
input_ids=input_ids,
positions=positions,
intermediate_tensors=intermediate_tensors,
inputs_embeds=inputs_embeds,
**kwargs,
)
@@ -1,454 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
from dataclasses import dataclass
from functools import partial
from typing import TYPE_CHECKING, Any
import torch
import torch.nn as nn
from vllm._aiter_ops import rocm_aiter_ops
from vllm.config.utils import getattr_iter
from vllm.distributed import get_dp_group, get_ep_group
from vllm.forward_context import ForwardContext, get_forward_context
from vllm.logger import init_logger
from vllm.model_executor.custom_op import PluggableLayer
from vllm.model_executor.layers.fused_moe import (
FusedMoEFactory,
MoERunner,
RoutedExperts,
)
from vllm.model_executor.models.interfaces import MixtureOfExperts
from vllm.model_executor.models.transformers.fuser import get_fuser
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
from .base import Base
from .utils import log_replacement, maybe_per_layer
if TYPE_CHECKING:
from vllm.config import VllmConfig
logger = init_logger(__name__)
@dataclass
class TransformersMoEState:
topk_ids: torch.Tensor | None = None
is_sequence_parallel: bool = False
# --8<-- [start:transformers_fused_moe]
@PluggableLayer.register("transformers_fused_moe")
class TransformersMoERunner(MoERunner):
"""Custom MoERunner for the Transformers modeling backend."""
# --8<-- [end:transformers_fused_moe]
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
super().__init__(*args, **kwargs)
self._moe_state = moe_state
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
def forward( # type: ignore[override]
self,
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
**kwargs: Any,
) -> torch.Tensor:
"""In Transformers `experts.forward` will have this signature, which is why
it deliberately does not match `MoERunner.forward`.
We discard any extra kwargs because we cannot use them here."""
# Note: we need to forward through a custom op so the topk_ids
# can be transferred without interfering with cudagraphs.
return torch.ops.vllm.transformers_moe_forward(
hidden_states,
topk_ids.to(torch.int32),
topk_weights.to(torch.float32),
self.layer_name,
)
def _forward_super(
self,
hidden_states: torch.Tensor,
topk_weights: torch.Tensor,
) -> torch.Tensor:
return super().forward(hidden_states, topk_weights)
def _transformers_moe_forward(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
"""Store the `topk_ids` in the layer and call the actual forward."""
forward_context: ForwardContext = get_forward_context()
self = forward_context.no_compile_layers[layer_name]
self._moe_state.topk_ids = topk_ids
return self._forward_super(hidden_states, topk_weights)
def _transformers_moe_forward_fake(
hidden_states: torch.Tensor,
topk_ids: torch.Tensor,
topk_weights: torch.Tensor,
layer_name: str,
) -> torch.Tensor:
return torch.empty_like(hidden_states)
direct_register_custom_op(
op_name="transformers_moe_forward",
op_func=_transformers_moe_forward,
mutates_args=["hidden_states"],
fake_impl=_transformers_moe_forward_fake,
tags=(torch.Tag.needs_fixed_stride_order,),
)
def _hf_topk_router_precomputed(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
moe_state: TransformersMoEState,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
weights arrive as `gating_output`, ids via `moe_state`."""
topk_weights = gating_output
topk_ids = moe_state.topk_ids
assert topk_ids is not None
# Handle all gather in expert parallel
if topk_ids.size(0) != hidden_states.size(0):
dp_metadata = get_forward_context().dp_metadata
assert dp_metadata is not None
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
assert sizes is not None
is_sp = moe_state.is_sequence_parallel
group = get_ep_group() if is_sp else get_dp_group()
assert sizes[group.rank_in_group] == topk_ids.shape[0]
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
return topk_weights, topk_ids
def _hf_topk_router(
hidden_states: torch.Tensor,
gating_output: torch.Tensor,
topk: int,
renormalize: bool,
gate: nn.Module,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
_, topk_weights, topk_ids = gate(hidden_states)
return topk_weights, topk_ids
class TransformersRoutedExperts(RoutedExperts):
def get_expert_mapping(
self,
ckpt_gate_proj_name: str | None = None,
ckpt_down_proj_name: str | None = None,
ckpt_up_proj_name: str | None = None,
include_fused: bool = False,
) -> list[tuple[str, str, int, str]]:
common_map = super().get_expert_mapping(
ckpt_gate_proj_name or "gate_proj",
ckpt_down_proj_name or "down_proj",
ckpt_up_proj_name or "up_proj",
include_fused,
)
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
if not include_fused:
return common_map + mixtral_map
common_fused, common_unfused = common_map[:3], common_map[3:]
mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:]
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
class MoEMixin(MixtureOfExperts, Base):
mlp_layers: list[nn.Module]
"""MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods."""
# Redeclared because `recursive_replace` assigns it below the first read
num_local_physical_experts: int
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
# Skip MixtureOfExperts.__init__ and call the next class in MRO
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
def update_physical_experts_metadata(
self,
num_physical_experts: int,
num_local_physical_experts: int,
):
assert self.num_local_physical_experts == num_local_physical_experts
self.num_physical_experts = num_physical_experts
self.num_local_physical_experts = num_local_physical_experts
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
for moe_block in self.mlp_layers:
moe_block.n_local_physical_experts = num_local_physical_experts
moe_block.n_physical_experts = num_physical_experts
moe_block.n_redundant_experts = self.num_redundant_experts
moe_block.experts.update_expert_map()
def recursive_replace(self):
"""Initialize the MoE layers."""
experts_name = "experts"
text_config = self.text_config
# Positional arguments
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
num_experts = getattr_iter(text_config, num_experts_names, None)
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
top_k = getattr_iter(text_config, top_k_names, None)
assert top_k is not None
hidden_size = text_config.hidden_size
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
assert intermediate_size is not None
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
# Common kwargs
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
# Routed scaling factor kwargs
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
# aiter applies routed_scaling_factor internally
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
routed_scaling_factor_kwargs = dict(
routed_scaling_factor=routed_scaling_factor,
apply_routed_scale_to_output=apply_routed_scale_to_output,
)
# Dtype the router computes in, if it is not the activation dtype.
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
if config_router_dtype is not None:
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
# Grouped topk routing kwargs
num_expert_group = getattr(text_config, "n_group", None)
topk_group = getattr(text_config, "topk_group", None)
use_grouped_topk = num_expert_group is not None and topk_group is not None
grouped_topk_routing_kwargs = dict(
use_grouped_topk=use_grouped_topk,
num_expert_group=num_expert_group,
topk_group=topk_group,
)
# MoE activation function
activation = "silu"
wrapped_arch = self.config.architectures[0].lower()
if "gptoss" in wrapped_arch:
activation = "swigluoai"
# Expert parallel load balancing kwargs
enable_eplb = self.parallel_config.enable_eplb
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
# MixtureOfExperts mixin settings
ep_size = get_ep_group().world_size
self.mlp_layers = []
self.moe_layers = []
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
self.num_logical_experts = num_experts
self.num_physical_experts = num_experts + num_redundant_experts
self.num_local_physical_experts = self.num_physical_experts // ep_size
self.num_routed_experts = num_experts
self.num_shared_experts = num_shared_experts
self.num_redundant_experts = num_redundant_experts
# Down projections of shared experts consumed by FusedMoE
shared_down_projs: list[tuple[nn.Module, str]] = []
# Recursively fuse MoE layers
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
qual_name = maybe_prefix(prefix, child_name)
# Naive implementations will have experts as ModuleList
is_modulelist = isinstance(child_module, nn.ModuleList)
# Packed implementations will have experts as 3D tensors of shapes like:
# gate_up_proj = (num_experts, 2 * intermediate_size, hidden_size)
# down_proj = (num_experts, intermediate_size, hidden_size)
params = list(child_module.parameters())
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
if child_name == experts_name and (is_modulelist or is_3d):
# Alias for readability
moe_block = module
experts = child_module
# Class of the fused block (parent of gate/experts/shared)
moe_block_cls = type(moe_block).__name__
experts_cls = type(experts).__name__
# Do the experts have biases
has_bias = False
for experts_param_name, _ in experts.named_parameters():
if "bias" in experts_param_name:
has_bias = True
break
# Whether the expert weights are stored in transposed format.
# Comes from Transformers `use_experts_implementation` decorator.
is_transposed = getattr(experts, "is_transposed", False)
# If the config does not specify num_shared_experts, but
# the model has shared experts, we assume there is one.
if self.num_shared_experts == 0:
for moe_block_param_name, _ in moe_block.named_parameters():
if "shared_expert" in moe_block_param_name:
self.num_shared_experts = 1
break
# Only pay for the layer index if something is per-layer;
# `extract_layer_index` raises for unnumbered module paths.
per_layer = (top_k, intermediate_size, norm_topk_prob)
layer_idx = (
extract_layer_index(qual_name)
if any(isinstance(v, list) for v in per_layer)
else 0
)
layer_top_k = maybe_per_layer(top_k, layer_idx)
layer_renormalize = maybe_per_layer(norm_topk_prob, layer_idx)
if layer_renormalize is None:
layer_renormalize = layer_top_k > 1
kwargs: dict[str, Any] = dict(
num_experts=num_experts,
top_k=layer_top_k,
hidden_size=hidden_size,
intermediate_size=maybe_per_layer(intermediate_size, layer_idx),
renormalize=layer_renormalize,
quant_config=self.quant_config,
prefix=qual_name,
activation=activation,
enable_eplb=enable_eplb,
num_redundant_experts=num_redundant_experts,
has_bias=has_bias,
is_fused_checkpoint_transposed=is_transposed,
routed_experts_cls=TransformersRoutedExperts,
)
# _maybe_apply_routed_scale_to_output edge case. Transformers
# decoder layers do not compensate for dividing by scaling factor.
divides_shared_output = (
routed_scaling_factor != 1.0
and apply_routed_scale_to_output
and self.model_config.dtype == torch.float16
)
fuser = MoEBlockFuser.match(
moe_block, experts_name, divides_shared_output
)
if fuser is not None:
# MoE block forward is fully replaced.
# gate/router and shared expert (if any) runs in MoERunner.
shared_experts = fuser.shared_experts(moe_block, prefix)
# Store shared experts for later down projection adjustment
if shared_experts is not None:
hf_shared = shared_experts.shared_experts
glu_fuser = get_fuser(hf_shared, GLUFuser)
if glu_fuser is not None:
down_name = glu_fuser.down_name
if down_name is not None:
shared_down_projs.append((hf_shared, down_name))
if scoring_func := fuser.scoring_func:
# Internal routing kwargs.
# Prefer config, otherwise read it from fuser.
router_dtype = config_router_dtype or fuser.router_dtype
gate = fuser.gate(moe_block, prefix, router_dtype)
kwargs |= dict(
scoring_func=scoring_func,
renormalize=(
layer_renormalize
if fuser.renormalize is None
else fuser.renormalize
),
gate=gate,
)
if router_dtype is not None:
kwargs["router_logits_dtype"] = router_dtype
if use_grouped_topk:
kwargs |= grouped_topk_routing_kwargs
if routed_scaling_factor != 1.0:
kwargs |= routed_scaling_factor_kwargs
bias = getattr(gate, "e_score_correction_bias", None)
if bias is not None:
kwargs["e_score_correction_bias"] = bias
else:
# External routing kwargs
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
_hf_topk_router,
gate=getattr(moe_block, fuser.gate_name),
),
)
kwargs |= dict(
is_sequence_parallel=(
self.parallel_config.use_sequence_parallel_moe
),
shared_experts=shared_experts,
)
fuser.rewrite_forward(moe_block)
logger.info_once(
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
" + shared experts" if fuser.shared_name else "",
moe_block_cls,
"external" if fuser.scoring_func is None else "internal",
)
else:
# MoE block forward is unmodified.
# gate/router and shared expert (if any) runs in Transformers.
# We then smuggle the topk_ids in using a custom op.
moe_state = TransformersMoEState()
kwargs |= dict(
num_expert_group=num_expert_group,
topk_group=topk_group,
custom_routing_function=partial(
_hf_topk_router_precomputed, moe_state=moe_state
),
runner_cls=TransformersMoERunner,
runner_args={"moe_state": moe_state},
)
logger.info_once(
"Fused: experts (%s) -> MoERunner (external routing)",
experts_cls,
)
fused_experts = FusedMoEFactory(**kwargs)
moe_block.experts = fused_experts
log_replacement(qual_name, experts, fused_experts)
# Update MixtureOfExperts mixin state
self.mlp_layers.append(moe_block)
self.moe_layers.append(fused_experts)
else:
_recursive_replace(child_module, prefix=qual_name)
_recursive_replace(self.model, prefix="model")
self.num_moe_layers = len(self.moe_layers)
# Continue with the replacement of layers in Base
super().recursive_replace()
# GLUFuser likely fused shared_experts. The down projection after the GLU
# normally immediately reduces but we want FusedMoE to handle the reduction.
for hf_shared, down_name in shared_down_projs:
hf_shared.get_submodule(down_name).reduce_results = False
File diff suppressed because it is too large Load Diff
@@ -1,407 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright 2024 The vLLM team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Transformers modeling backend utilities."""
from collections.abc import Callable, Iterator
from contextlib import contextmanager
from itertools import chain
from operator import attrgetter
from pathlib import Path
from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar
import torch
from torch import nn
from vllm.logger import init_logger
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
from vllm.model_executor.layers.layernorm import LayerNorm
from vllm.model_executor.layers.linear import (
ColumnParallelLinear,
ReplicatedLinear,
RowParallelLinear,
)
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.models.utils import maybe_prefix
from vllm.transformers_utils.config import iter_rope_parameters
if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.model_executor.layers.quantization import QuantizationConfig
logger = init_logger(__name__)
# Copied from `accelerate`
@contextmanager
def init_on_device_without_buffers(device: torch.device):
"""A context manager under which models are initialized with all
parameters on the specified device. However buffers are not
initialized on specified device.
Args:
device (`torch.device`):
Device to initialize all parameters on.
"""
old_register_parameter = nn.Module.register_parameter
def register_empty_parameter(module, name, param):
old_register_parameter(module, name, param)
if param is not None:
param_cls = type(module._parameters[name])
kwargs = module._parameters[name].__dict__
kwargs["requires_grad"] = param.requires_grad
module._parameters[name] = param_cls(
module._parameters[name].to(device), **kwargs
)
tensor_constructors_to_patch: dict[str, Callable] = {}
def patch_tensor_constructor(fn):
def wrapper(*args, **kwargs):
kwargs["device"] = device
return fn(*args, **kwargs)
return wrapper
try:
nn.Module.register_parameter = register_empty_parameter
for torch_function_name in tensor_constructors_to_patch:
setattr(
torch,
torch_function_name,
patch_tensor_constructor(getattr(torch, torch_function_name)),
)
yield
finally:
nn.Module.register_parameter = old_register_parameter
for (
torch_function_name,
old_torch_function,
) in tensor_constructors_to_patch.items():
setattr(torch, torch_function_name, old_torch_function)
Style = Literal[
"colwise",
"rowwise",
"replicate",
"colwise_gather_output",
"rowwise_split_input",
]
def replace_linear_class(
linear: nn.Linear,
style: Style = "replicate",
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
"""Replace nn.Linear with one of vLLM's tensor parallel linear classes.
Args:
linear: `nn.Linear` to be replaced.
style: Tensor parallel style of the new linear, e.g. "colwise".
quant_config: Quantization config for the new linear.
prefix: Module prefix of the new linear, used for quantization lookup.
Returns:
The new linear.
"""
if not isinstance(style, str):
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
vllm_linear_cls, vllm_linear_kwargs = {
"colwise": (ColumnParallelLinear, {}),
"rowwise": (RowParallelLinear, {}),
"replicate": (ReplicatedLinear, {}),
"colwise_gather_output": (ColumnParallelLinear, {"gather_output": True}),
"rowwise_split_input": (RowParallelLinear, {"input_is_parallel": False}),
}.get(style, (ReplicatedLinear, {}))
return vllm_linear_cls(
input_size=linear.in_features,
output_size=linear.out_features,
bias=linear.bias is not None,
quant_config=quant_config,
prefix=prefix,
return_bias=False,
**vllm_linear_kwargs,
)
TorchConv: TypeAlias = nn.Conv2d | nn.Conv3d
VllmConv: TypeAlias = Conv2dLayer | Conv3dLayer
def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
"""Replace a Transformers Conv2d/Conv3d with vLLM's Conv2d/Conv3d.
Args:
conv: `nn.Conv2d` or `nn.Conv3d` to be replaced.
Returns:
The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported,
returns the original conv module.
"""
# vLLM does not handle non-zero padding modes
if conv.padding_mode != "zeros":
return conv
vllm_conv_cls = {
nn.Conv2d: Conv2dLayer,
nn.Conv3d: Conv3dLayer,
}.get(type(conv))
if vllm_conv_cls is None:
return conv
return vllm_conv_cls(
in_channels=conv.in_channels,
out_channels=conv.out_channels,
kernel_size=conv.kernel_size,
stride=conv.stride,
padding=conv.padding,
dilation=conv.dilation,
groups=conv.groups,
bias=conv.bias is not None,
padding_mode=conv.padding_mode,
params_dtype=conv.weight.dtype,
)
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
`LayerNorm`.
Args:
layernorm: `nn.LayerNorm` to be replaced.
Returns:
The new `LayerNorm`. If the layernorm is not supported (a subclass with
its own behavior, multi-dim `normalized_shape`, or not both
elementwise-affine and biased), returns the original module unchanged.
"""
if (
type(layernorm) is not nn.LayerNorm
or len(layernorm.normalized_shape) != 1
or not layernorm.elementwise_affine
or layernorm.bias is None
):
return layernorm
return LayerNorm(
layernorm.normalized_shape[0],
eps=layernorm.eps,
dtype=layernorm.weight.dtype,
)
def attrsetter(attr: str) -> Callable[[object, object], None]:
"""Set a possibly nested attribute, like the inverse of attrgetter."""
parent, _, name = attr.rpartition(".")
def setter(obj: object, value: object):
attr_parent = attrgetter(parent)(obj) if parent else obj
setattr(attr_parent, name, value)
return setter
class _UninitializedEmbedding(nn.Embedding):
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
def __init__(self):
pass
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
_rebased_embedding_classes: dict[type[nn.Embedding], type[VocabParallelEmbedding]] = {}
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
Args:
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
maps to the same class.
Returns:
The new class, to assign to `__class__` of an instance of `cls`.
"""
if cls not in _rebased_embedding_classes:
rebased_cls = type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
_rebased_embedding_classes[cls] = rebased_cls
return _rebased_embedding_classes[cls]
def replace_embedding_class(
embedding: nn.Embedding,
quant_config: "QuantizationConfig | None" = None,
*,
prefix: str = "",
) -> nn.Module:
"""Replace `nn.Embedding` with `VocabParallelEmbedding`.
Args:
embedding: The `nn.Embedding` holding a vocab table.
quant_config: Quantization config for the new embedding.
prefix: Qualname of `embedding`, used to look up its quantization method.
Returns:
The module to install in `embedding`'s place.
"""
# Tied tables are reached more than once, so don't rebase an already new embedding
if isinstance(embedding, VocabParallelEmbedding):
return embedding
kwargs = dict(
num_embeddings=embedding.num_embeddings,
embedding_dim=embedding.embedding_dim,
params_dtype=embedding.weight.dtype,
quant_config=quant_config,
prefix=prefix,
)
# If `embedding` is a bare `nn.Embedding`, simple replace
if type(embedding) is nn.Embedding:
return VocabParallelEmbedding(**kwargs)
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
VocabParallelEmbedding.__init__(embedding, **kwargs)
return embedding
def recursive_replace_linear(
model: nn.Module,
quant_config: "QuantizationConfig | None",
prefix: str = "",
):
"""Recursively replace linear modules in the model as needed."""
def _recursive_replace(module: nn.Module, prefix: str):
for child_name, child_module in module.named_children():
new_module = child_module
qual_name = maybe_prefix(prefix, child_name)
# Replace modules as needed
if isinstance(child_module, nn.Linear):
new_module = replace_linear_class(
child_module,
"replicate",
quant_config,
prefix=qual_name,
)
else:
_recursive_replace(child_module, prefix=qual_name)
if new_module is not child_module:
setattr(module, child_name, new_module)
_recursive_replace(model, prefix=prefix)
T = TypeVar("T")
def maybe_per_layer(value: T | list[T], layer_idx: int) -> T:
"""Pick `layer_idx`'s entry from a config field that may be sized per layer."""
return value[layer_idx] if isinstance(value, list) else value
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
"""`module`'s own state (i.e. named parameters and buffers)."""
return chain(module.named_parameters(), module.named_buffers())
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
logger.debug("%s: %s -> %s", name, old_module, new_module)
def get_feature_request_tip(
model: str,
trust_remote_code: bool,
) -> str:
hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new"
gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose"
url = hf_url if trust_remote_code else gh_url
prefix = f"Please open {url} to request support for this feature. "
if Path(model).exists():
prefix = ""
doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models"
tip = f"See {doc_url} for instructions on how to add support yourself."
return f"{prefix}{tip}"
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
if max_position_embeddings is None:
return False
return vllm_config.model_config.max_model_len <= max_position_embeddings
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
Defaults to `True` but is disabled in the following situations:
- The model uses dynamic rope scaling that can still rescale within
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
"""
text_config = vllm_config.model_config.hf_config.get_text_config()
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
if "dynamic" not in rope_types:
return True
# Layer types using longrope still sync, even once dynamic rope is disabled
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
The update syncs on `torch.max(position_ids)` every forward, which breaks
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
"default" skips it while keeping the `inv_freq` computed at init.
"""
if not dynamic_rope_update_is_noop(vllm_config):
return
for module in model.modules():
rope_type = getattr(module, "rope_type", None)
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
continue
if isinstance(rope_type, dict):
module.rope_type = {
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
}
elif rope_type == "dynamic":
module.rope_type = "default"