[Mini-VLLM] Cut dead subsystems across serving, connectors, parsers
- Remove P/D transfer connectors: nixl, nixl_chat, mooncake x2, hf3fs, moriio, hisparse, example and hidden-states connectors - Reduce weight_transfer to the base API used by model loading - Remove kv_offload tiering, simple_kv_offload, ec cpu/example, snapshot command, engine snapshot server - Remove transformers model fallback, dsv4 tokenizers, dead platforms cpu/tpu/xpu, flashmla third_party, dsa, megamoe/flydsl, dead attn ops, dead layers (short_conv, fused_embed_norm, indexer_topk, relu2 gemma_ar, lightning_attn, minimax_rms_norm), gemm_rs_ar - Keep MoE LoRA op (imported eagerly by punica_gpu); drop fp8 variant - Rebuild parser hubs around Qwen3 adapters only (36 dead adapters) - Remove dead tests with the code they covered Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -115,77 +115,9 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
"Qwen3_5MoeMTP": ("qwen3_5_mtp", "Qwen3_5MoeMTP"),
|
||||
}
|
||||
|
||||
_TRANSFORMERS_SUPPORTED_MODELS = {
|
||||
# Text generation models
|
||||
"ArceeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"CwmForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"FlexOlmoForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"GlmForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"GPTBigCodeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"GPTNeoXForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanDenseV1ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"HunYuanMoEV1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"Jais2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Kolibri1ForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"MellumForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
"NanbeigeForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"OlmoForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Olmo2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Olmo3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"PhiForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SeedOssForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"SmolLM3ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"Starcoder2ForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"VaultGemmaForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
# Multimodal models
|
||||
"Emu3ForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"HunYuanVLForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"VibeVoiceAsrForConditionalGeneration": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
}
|
||||
_TRANSFORMERS_SUPPORTED_MODELS = {}
|
||||
|
||||
_TRANSFORMERS_BACKEND_MODELS = {
|
||||
# Text generation models
|
||||
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
"TransformersMoEForCausalLM": ("transformers", "TransformersMoEForCausalLM"),
|
||||
# Multimodal models
|
||||
"TransformersMultiModalForCausalLM": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForCausalLM",
|
||||
),
|
||||
"TransformersMultiModalMoEForCausalLM": (
|
||||
"transformers",
|
||||
"TransformersMultiModalMoEForCausalLM",
|
||||
),
|
||||
# Embedding models
|
||||
"TransformersEmbeddingModel": ("transformers", "TransformersEmbeddingModel"),
|
||||
"TransformersMoEEmbeddingModel": ("transformers", "TransformersMoEEmbeddingModel"),
|
||||
"TransformersMultiModalEmbeddingModel": (
|
||||
"transformers",
|
||||
"TransformersMultiModalEmbeddingModel",
|
||||
),
|
||||
# Sequence classification models
|
||||
"TransformersForSequenceClassification": (
|
||||
"transformers",
|
||||
"TransformersForSequenceClassification",
|
||||
),
|
||||
"TransformersMoEForSequenceClassification": (
|
||||
"transformers",
|
||||
"TransformersMoEForSequenceClassification",
|
||||
),
|
||||
"TransformersMultiModalForSequenceClassification": (
|
||||
"transformers",
|
||||
"TransformersMultiModalForSequenceClassification",
|
||||
),
|
||||
}
|
||||
_TRANSFORMERS_BACKEND_MODELS = {}
|
||||
|
||||
_VLLM_MODELS = {
|
||||
**_TEXT_GENERATION_MODELS,
|
||||
|
||||
@@ -1,155 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Wrapper around `transformers` models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch.nn.functional as F
|
||||
from transformers.modeling_utils import ALL_ATTENTION_FUNCTIONS
|
||||
|
||||
from vllm.model_executor.models.transformers.base import VLLM_ATTN_ATTR
|
||||
from vllm.model_executor.models.transformers.causal import CausalMixin
|
||||
from vllm.model_executor.models.transformers.moe import MoEMixin
|
||||
from vllm.model_executor.models.transformers.multimodal import (
|
||||
MultiModalDummyInputsBuilder,
|
||||
MultiModalMixin,
|
||||
MultiModalProcessingInfo,
|
||||
MultiModalProcessor,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
|
||||
if TYPE_CHECKING:
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.attention import Attention, MLAAttention
|
||||
|
||||
|
||||
def check_sinks(
|
||||
module: "torch.nn.Module",
|
||||
self_attn: "Attention | MLAAttention",
|
||||
s_aux: "torch.Tensor | None",
|
||||
):
|
||||
"""Fail loudly if the model applies a sink the attention layer will not.
|
||||
|
||||
Only the attention impl can fold a sink into the softmax denominator, so a sink
|
||||
that never reached `Attention` is dropped and every softmax is subtly wrong.
|
||||
"""
|
||||
if s_aux is None or getattr(self_attn, "has_sink", False):
|
||||
return
|
||||
raise ValueError(
|
||||
f"{type(module).__name__} applies attention sinks, but they were not passed "
|
||||
f"to {type(self_attn).__name__}, so the output would be wrong. Either the "
|
||||
"Transformers modeling backend could not find the parameter holding them, or "
|
||||
"vLLM does not support sinks for this kind of attention. Please open an issue "
|
||||
"at https://github.com/vllm-project/vllm/issues/new"
|
||||
)
|
||||
|
||||
|
||||
def vllm_attention_forward(
|
||||
# Transformers args
|
||||
module: "torch.nn.Module",
|
||||
query: "torch.Tensor",
|
||||
key: "torch.Tensor",
|
||||
value: "torch.Tensor",
|
||||
attention_mask: "torch.Tensor",
|
||||
**kwargs,
|
||||
):
|
||||
self_attn = getattr(module, VLLM_ATTN_ATTR)
|
||||
check_sinks(module, self_attn, kwargs.get("s_aux"))
|
||||
hidden = query.shape[-2]
|
||||
head_dim_qk = query.shape[-1]
|
||||
head_dim_v = value.shape[-1]
|
||||
query, key, value = (x.transpose(1, 2) for x in (query, key, value))
|
||||
query, key, value = (x.reshape(hidden, -1) for x in (query, key, value))
|
||||
# Pad `value` up to the query/key head size when it is smaller (expanded
|
||||
# MLA). A larger last dim just means `value` isn't split per head, e.g.
|
||||
# packed grouped/multi-query projections, and needs no padding.
|
||||
pad_value = head_dim_v < head_dim_qk
|
||||
if pad_value:
|
||||
value = F.pad(value.view(-1, head_dim_v), (0, head_dim_qk - head_dim_v))
|
||||
value = value.reshape(hidden, -1)
|
||||
attn_output = self_attn.forward(query, key, value)
|
||||
if pad_value:
|
||||
attn_output = attn_output.view(-1, head_dim_qk)[..., :head_dim_v]
|
||||
attn_output = attn_output.reshape(hidden, -1)
|
||||
return attn_output, None
|
||||
|
||||
|
||||
def vllm_mla_attention_forward(
|
||||
# Transformers args
|
||||
module: "torch.nn.Module",
|
||||
query: "torch.Tensor",
|
||||
kv_c_normed: "torch.Tensor",
|
||||
k_pe: "torch.Tensor",
|
||||
attention_mask: "torch.Tensor",
|
||||
**kwargs,
|
||||
):
|
||||
self_attn = getattr(module, VLLM_ATTN_ATTR)
|
||||
check_sinks(module, self_attn, kwargs.get("s_aux"))
|
||||
# [batch=1, heads, num_tokens, qk_head_dim] -> [num_tokens, heads, qk_head_dim]
|
||||
query = query.transpose(1, 2).flatten(0, 1)
|
||||
num_tokens, num_heads = query.shape[:2]
|
||||
# [batch=1, num_tokens, kv_lora_rank] -> [num_tokens, kv_lora_rank]
|
||||
kv_c_normed = kv_c_normed.reshape(-1, kv_c_normed.shape[-1])
|
||||
# [batch=1, heads=1, num_tokens, qk_rope] -> [num_tokens, 1, qk_rope]
|
||||
k_pe = k_pe.reshape(-1, 1, k_pe.shape[-1])
|
||||
attn_output = self_attn.forward(
|
||||
query,
|
||||
kv_c_normed,
|
||||
k_pe,
|
||||
output_shape=(num_tokens, num_heads * self_attn.v_head_dim),
|
||||
)
|
||||
return attn_output, None
|
||||
|
||||
|
||||
ALL_ATTENTION_FUNCTIONS.register("vllm", vllm_attention_forward)
|
||||
ALL_ATTENTION_FUNCTIONS.register("vllm_mla", vllm_mla_attention_forward)
|
||||
|
||||
|
||||
# Text only models
|
||||
class TransformersForCausalLM(CausalMixin): ...
|
||||
|
||||
|
||||
class TransformersMoEForCausalLM(MoEMixin, CausalMixin): ...
|
||||
|
||||
|
||||
# Multimodal models
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MultiModalProcessor,
|
||||
info=MultiModalProcessingInfo,
|
||||
dummy_inputs=MultiModalDummyInputsBuilder,
|
||||
)
|
||||
class TransformersMultiModalForCausalLM(MultiModalMixin, CausalMixin): ...
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
MultiModalProcessor,
|
||||
info=MultiModalProcessingInfo,
|
||||
dummy_inputs=MultiModalDummyInputsBuilder,
|
||||
)
|
||||
class TransformersMultiModalMoEForCausalLM(MoEMixin, MultiModalMixin, CausalMixin): ...
|
||||
|
||||
|
||||
def __getattr__(name: str):
|
||||
"""Handle imports of non-existent classes with a helpful error message."""
|
||||
if name not in globals():
|
||||
raise AttributeError(
|
||||
"The Transformers modeling backend does not currently have a class to "
|
||||
f"handle the requested model type: {name}. Please open an issue at "
|
||||
"https://github.com/vllm-project/vllm/issues/new"
|
||||
)
|
||||
return globals()[name]
|
||||
@@ -1,860 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend base class."""
|
||||
|
||||
import os
|
||||
from collections.abc import Callable, Iterable
|
||||
from contextlib import contextmanager
|
||||
from functools import cached_property
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
from typing import TYPE_CHECKING, Any, NamedTuple
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from torch import nn
|
||||
from transformers import AutoModel, PreTrainedConfig
|
||||
from transformers.conversion_mapping import (
|
||||
WeightRenaming,
|
||||
get_checkpoint_conversion_mapping,
|
||||
get_model_conversion_mapping,
|
||||
)
|
||||
|
||||
from vllm.compilation.decorators import DynamicArgDims, support_torch_compile
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_pp_group, get_tp_group
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention import Attention, EncoderOnlyAttention
|
||||
from vllm.model_executor.layers.attention_layer_base import AttentionLayerBase
|
||||
from vllm.model_executor.layers.fused_moe import MoERunner
|
||||
from vllm.model_executor.models.interfaces import (
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsQuant,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModel
|
||||
from vllm.model_executor.models.transformers.fuser import BaseFuser, Fusers
|
||||
from vllm.model_executor.models.transformers.fusers import AttentionFuser, MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers.attention import (
|
||||
VLLM_ATTN_IMPL,
|
||||
VLLM_MLA_ATTN_IMPL,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
attrsetter,
|
||||
can_enable_torch_compile,
|
||||
disable_noop_dynamic_rope,
|
||||
get_feature_request_tip,
|
||||
init_on_device_without_buffers,
|
||||
log_replacement,
|
||||
named_state,
|
||||
replace_conv_class,
|
||||
replace_embedding_class,
|
||||
replace_layernorm_class,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import (
|
||||
AutoWeightsLoader,
|
||||
PPMissingLayer,
|
||||
ShardId,
|
||||
WeightsMapper,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
maybe_prefix,
|
||||
)
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from transformers import PreTrainedModel
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
try:
|
||||
from vllm.model_executor.layers.attention.mla_attention import MLAAttention
|
||||
except ImportError: # MLA support removed from this build
|
||||
MLAAttention = type("AbsentMLAAttention", (), {})
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
VLLM_ATTN_ATTR = "attn"
|
||||
"""Attribute in which to store the vLLM `Attention` instance that serves a Transformers
|
||||
attention module. `"attn"` is chosen deliberately to match the mappings generated by
|
||||
`maybe_remap_kv_scale_name`."""
|
||||
|
||||
|
||||
class PreTrainedModelClasses(NamedTuple):
|
||||
decoder: type["PreTrainedModel"]
|
||||
encoders: dict[str, type["PreTrainedModel"]]
|
||||
"""Modality -> encoder class, for each modality that has one."""
|
||||
|
||||
|
||||
class Base(
|
||||
nn.Module,
|
||||
VllmModel,
|
||||
SupportsQuant,
|
||||
SupportsLoRA,
|
||||
SupportsPP,
|
||||
SupportsEagle,
|
||||
SupportsEagle3,
|
||||
):
|
||||
hf_to_vllm_mapper: WeightsMapper
|
||||
|
||||
# TODO transformers will have a util to get these
|
||||
embedding_modules = {"embed_tokens": "input_embeddings"}
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
logger.info("Using Transformers modeling backend.")
|
||||
|
||||
self.vllm_config = vllm_config
|
||||
self.config = vllm_config.model_config.hf_config
|
||||
self.text_config = self.config.get_text_config()
|
||||
self.cache_config = vllm_config.cache_config
|
||||
self.compilation_config = vllm_config.compilation_config
|
||||
self.device_config = vllm_config.device_config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.parallel_config = vllm_config.parallel_config
|
||||
self.quant_config = vllm_config.quant_config
|
||||
|
||||
self.pp_group = get_pp_group()
|
||||
self.tp_group = get_tp_group()
|
||||
|
||||
# Attrs for weight loading (see self.load_weights)
|
||||
self.ignore_unexpected_prefixes: list[str] = []
|
||||
"""Ignore unexpected weights whose qualname starts with these prefixes."""
|
||||
self.ignore_unexpected_suffixes: list[str] = []
|
||||
"""Ignore unexpected weights whose qualname ends with these suffixes."""
|
||||
self.packed_modules_mapping: dict[str, list[str]] = {}
|
||||
"""Fused module -> constituent projections, populated by `recursive_replace`
|
||||
for the quantization machinery and loaders (e.g. bitsandbytes)."""
|
||||
self.fusers: dict[str, list[BaseFuser]] = {}
|
||||
"""Module qualname -> the fusers applied to it, populated
|
||||
by `recursive_replace` for `_create_attention_instances`."""
|
||||
self.attention_fusers: dict[int, tuple[str, AttentionFuser]] = {}
|
||||
"""`layer_idx` -> the qualname and fuser of the module computing that
|
||||
layer's attention, populated by `recursive_replace` for
|
||||
`_create_attention_instances`."""
|
||||
|
||||
# Attrs for Eagle3 (see self.set_aux_hidden_state_layers)
|
||||
self._target_class: type[nn.Module] = nn.Module
|
||||
"""Target class for Eagle3 aux hidden state recording."""
|
||||
self._layer_names: dict[int, str] = {}
|
||||
"""Mapping from layer index to layer name for Eagle3."""
|
||||
self._output_aux_hidden_states_kwargs: dict[str, bool] = {}
|
||||
"""Kwargs to pass to model forward for Eagle3 aux hidden states."""
|
||||
|
||||
if self.quant_config:
|
||||
quant_method_name = self.quant_config.get_name()
|
||||
# Check for unsupported quantization methods.
|
||||
if quant_method_name in ("mxfp4", "gpt_oss_mxfp4"):
|
||||
raise NotImplementedError(
|
||||
"Transformers modeling backend does "
|
||||
"not support MXFP4 quantization yet."
|
||||
)
|
||||
|
||||
self._patch_config()
|
||||
self._decorate_for_torch_compile()
|
||||
# Init on "meta" to delay allocating GPU tensors
|
||||
with (
|
||||
self._mark_model_components(vllm_config),
|
||||
init_on_device_without_buffers("meta"),
|
||||
):
|
||||
from_config_kwargs = self._from_config_kwargs
|
||||
self.model: PreTrainedModel = AutoModel.from_config(**from_config_kwargs)
|
||||
|
||||
# Create weight name to module qualname mapper
|
||||
self._create_hf_to_vllm_mapper()
|
||||
# Remove layers not on this pipeline parallel rank
|
||||
self.pipeline_parallel()
|
||||
# Skip the device sync of dynamic rope where it can never rescale
|
||||
disable_noop_dynamic_rope(self.model.get_decoder(), vllm_config)
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
self._create_attention_instances(prefix)
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
|
||||
# Upcast weights Transformers always keeps in fp32
|
||||
self.keep_in_fp32(self.model)
|
||||
|
||||
# Pipeline parallel intermediate tensors
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.text_config.hidden_size
|
||||
)
|
||||
|
||||
def _patch_config(self):
|
||||
"""Patch the config to ensure that the model is created correctly:
|
||||
|
||||
- Sets the attention implementation to "vllm" so the attention instances from
|
||||
`_create_attention_instances` are used
|
||||
- Sets the dtype to the default torch dtype set by vLLM because Transformers
|
||||
uses the config dtype when creating the model
|
||||
"""
|
||||
self.text_config._attn_implementation = VLLM_ATTN_IMPL
|
||||
self.config.dtype = torch.get_default_dtype()
|
||||
|
||||
@contextmanager
|
||||
def _mark_model_components(self, vllm_config: "VllmConfig"):
|
||||
"""Mark language model and tower submodules as `self.model` is created.
|
||||
|
||||
Nothing to do in `Base`, `MultiModalMixin` will override."""
|
||||
yield
|
||||
|
||||
@cached_property
|
||||
def _from_config_kwargs(self) -> dict[str, Any]:
|
||||
"""The kwargs used to create `self.model`."""
|
||||
return dict(
|
||||
config=self.config,
|
||||
dtype=self.model_config.dtype,
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
def _find_encoder_classes(
|
||||
self, model: "PreTrainedModel"
|
||||
) -> dict[str, type["PreTrainedModel"]]:
|
||||
"""Find the encoder class of each modality `model` has one for.
|
||||
|
||||
Text models have none. Multi-modal ones override this.
|
||||
"""
|
||||
return {}
|
||||
|
||||
@cached_property
|
||||
def _pre_trained_model_classes(self) -> PreTrainedModelClasses:
|
||||
"""The decoder class and the encoder class of each modality that has one.
|
||||
|
||||
Both come from a single throwaway model, since building one is not cheap.
|
||||
"""
|
||||
with torch.device("meta"):
|
||||
model: PreTrainedModel = AutoModel.from_config(**self._from_config_kwargs)
|
||||
model_classes = PreTrainedModelClasses(
|
||||
decoder=type(model.get_decoder()),
|
||||
encoders=self._find_encoder_classes(model),
|
||||
)
|
||||
del model
|
||||
|
||||
logger.debug("Identified model classes as: %s", model_classes)
|
||||
return model_classes
|
||||
|
||||
def _decorate_cls_for_torch_compile(
|
||||
self,
|
||||
cls: type["PreTrainedModel"],
|
||||
dynamic_arg_dims: DynamicArgDims | None,
|
||||
enable_if: Callable[["VllmConfig"], bool],
|
||||
is_encoder: bool,
|
||||
):
|
||||
"""Decorate `cls` to indicate to vLLM that it supports torch compile.
|
||||
|
||||
Args:
|
||||
cls: The PreTrainedModel class to decorate.
|
||||
dynamic_arg_dims: A mapping from argument name to the dynamic dimensions
|
||||
of the argument. If None, default dynamic arg dims will be used. See
|
||||
[`support_torch_compile`][vllm.compilation.decorators.support_torch_compile]
|
||||
for more details.
|
||||
enable_if: A function which takes in the vLLM config and returns whether
|
||||
torch compile should be enabled for this class.
|
||||
is_encoder: Whether the class being decorated is an encoder.
|
||||
|
||||
"""
|
||||
logger.debug(
|
||||
"Decorating `%s` as %s for torch compile with dynamic_arg_dims of %s",
|
||||
cls.__name__,
|
||||
"encoder" if is_encoder else "decoder",
|
||||
dynamic_arg_dims,
|
||||
)
|
||||
|
||||
support_torch_compile(
|
||||
dynamic_arg_dims=dynamic_arg_dims,
|
||||
enable_if=enable_if,
|
||||
is_encoder=is_encoder,
|
||||
use_aot_compile=False,
|
||||
)(cls)
|
||||
|
||||
def _decorate_for_torch_compile(self):
|
||||
"""Decorate the model's decoder class to indicate to vLLM that it
|
||||
supports torch compile if `can_enable_torch_compile` is True."""
|
||||
self._decorate_cls_for_torch_compile(
|
||||
cls=self._pre_trained_model_classes.decoder,
|
||||
# Applied to a PreTrainedModel so the batch dimension will exist
|
||||
dynamic_arg_dims=DynamicArgDims(
|
||||
input_ids=1, # shape: [1, seq_len]
|
||||
inputs_embeds=1, # shape: [1, seq_len, hidden_size]
|
||||
position_ids=-1, # shape: [1, seq_len] or [3, 1, seq_len] for mrope
|
||||
),
|
||||
enable_if=can_enable_torch_compile,
|
||||
is_encoder=False,
|
||||
)
|
||||
|
||||
def _create_hf_to_vllm_mapper(self):
|
||||
"""Create a WeightsMapper to map checkpoint weight names to module qualnames.
|
||||
|
||||
This handles:
|
||||
|
||||
- Transformers weight renaming from `WeightRenaming`
|
||||
- Checkpoints saved with a base model prefix that is not `model`
|
||||
- Checkpoints saved with no base model prefix
|
||||
- Any quantization config specific mappings
|
||||
"""
|
||||
orig_to_new_renaming: list[WeightRenaming] = []
|
||||
orig_to_new_regex: dict[re.Pattern, str | None] = {}
|
||||
|
||||
mappings = get_model_conversion_mapping(self.model)
|
||||
# Ensure layers that are part of the model, but not AutoModel, get their mapping
|
||||
instantiated = {type(m).__name__ for m in self.model.modules()}
|
||||
for arch in self.config.architectures or []:
|
||||
if arch not in instantiated:
|
||||
mappings.extend(get_checkpoint_conversion_mapping(arch) or [])
|
||||
|
||||
for mapping in mappings:
|
||||
# Handle weights which have been renamed in Transformers
|
||||
if isinstance(mapping, WeightRenaming):
|
||||
orig_to_new_renaming.append(mapping)
|
||||
# TODO: Handle WeightConverter to enable layer merging
|
||||
|
||||
# Handle unexpected weights which should be ignored
|
||||
if self.model._keys_to_ignore_on_load_unexpected is not None:
|
||||
for key in self.model._keys_to_ignore_on_load_unexpected:
|
||||
orig_to_new_regex[re.compile(key)] = None
|
||||
|
||||
# Standardise base model prefix
|
||||
bmp = self.model.base_model_prefix
|
||||
expected_bmp = r"model.\1"
|
||||
# Handle checkpoints saved with different base model prefix
|
||||
if bmp and bmp != "model":
|
||||
different_bmp_pattern = re.compile(rf"^{bmp}\.(.+)")
|
||||
orig_to_new_regex[different_bmp_pattern] = expected_bmp
|
||||
# Handle direct children of self.model which were saved without the model prefix
|
||||
direct_children = chain(
|
||||
self.model.named_children(),
|
||||
self.model.named_parameters(recurse=False),
|
||||
self.model.named_buffers(recurse=False),
|
||||
)
|
||||
model_children = "|".join(name for name, _ in direct_children)
|
||||
missing_bmp_pattern = re.compile(rf"^(?!model\.)(({model_children}).*)")
|
||||
orig_to_new_regex[missing_bmp_pattern] = expected_bmp
|
||||
# Handle weights saved as direct children of self.model which no longer are
|
||||
unexpected_bmp_pattern = re.compile(rf"^(model\.)((?!{model_children}).+)")
|
||||
orig_to_new_regex[unexpected_bmp_pattern] = r"\2"
|
||||
# Handle lm_head which was saved inside the base model
|
||||
nested_lm_head_pattern = re.compile(r"^model\.(.+\.)*(lm_head.+)")
|
||||
orig_to_new_regex[nested_lm_head_pattern] = r"\2"
|
||||
|
||||
self.hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_renaming=orig_to_new_renaming,
|
||||
orig_to_new_regex=orig_to_new_regex,
|
||||
)
|
||||
|
||||
# Apply mapping to quantization config if needed
|
||||
self._maybe_apply_model_mapping()
|
||||
|
||||
def _get_tie_word_embeddings(self):
|
||||
"""Check if the model has tied word embeddings."""
|
||||
# Models created with Transformers v4 and v5 will store this in different places
|
||||
tie_word_embeddings_v4 = getattr(self.text_config, "tie_word_embeddings", False)
|
||||
tie_word_embeddings_v5 = getattr(self.config, "tie_word_embeddings", False)
|
||||
return tie_word_embeddings_v4 or tie_word_embeddings_v5
|
||||
|
||||
def pipeline_parallel(self):
|
||||
"""Apply the model's pipeline parallelization plan."""
|
||||
if self.pp_group.world_size <= 1:
|
||||
return
|
||||
|
||||
if self.model.supports_pp_plan:
|
||||
module = self.model
|
||||
names = list(module._pp_plan.keys())
|
||||
else:
|
||||
module = self.model.get_decoder()
|
||||
has_parameters = lambda m: next(m.parameters(), None) is not None
|
||||
names = [n for n, c in module.named_children() if has_parameters(c)]
|
||||
tip = get_feature_request_tip(
|
||||
self.model_config.model, self.model_config.trust_remote_code
|
||||
)
|
||||
logger.warning_once(
|
||||
"%s does not define a pipeline parallel plan. The Transformers "
|
||||
"modeling backend will infer the split from the layers of %s in order "
|
||||
"of declaration and keep parameter-free modules on every rank. This "
|
||||
"may fail if the model's structure is non-standard. %s",
|
||||
type(self.model).__name__,
|
||||
type(module).__name__,
|
||||
tip,
|
||||
)
|
||||
|
||||
module_lists = []
|
||||
module_list_idx = None
|
||||
for i, name in enumerate(names):
|
||||
# attrgetter in case the module is nested (e.g. "text_model.layers")
|
||||
if isinstance(attrgetter(name)(module), nn.ModuleList):
|
||||
module_lists.append(name)
|
||||
module_list_idx = i
|
||||
|
||||
if len(module_lists) > 1:
|
||||
raise ValueError(
|
||||
"Pipeline parallel of models with multiple `ModuleList`s "
|
||||
"in the base model are not supported yet!"
|
||||
)
|
||||
if module_list_idx is None:
|
||||
raise ValueError(f"Could not find `ModuleList` in {type(module)}")
|
||||
|
||||
# Layers before module list
|
||||
for name in names[:module_list_idx]:
|
||||
if self.pp_group.is_first_rank or (
|
||||
self._get_tie_word_embeddings() and self.pp_group.is_last_rank
|
||||
):
|
||||
continue
|
||||
# attrsetter in case the module is nested (e.g. "text_model.embed_tokens")
|
||||
attrsetter(name)(module, PPMissingLayer())
|
||||
|
||||
# Module list
|
||||
start_layer, end_layer = get_pp_indices(
|
||||
self.text_config.num_hidden_layers,
|
||||
self.pp_group.rank_in_group,
|
||||
self.pp_group.world_size,
|
||||
)
|
||||
layers_name = names[module_list_idx]
|
||||
# attrgetter in case the module is nested (e.g. "text_model.layers")
|
||||
layers = attrgetter(layers_name)(module)
|
||||
for i in range(len(layers)):
|
||||
if start_layer <= i and i < end_layer:
|
||||
continue
|
||||
layers[i] = PPMissingLayer()
|
||||
|
||||
# Layers after module list
|
||||
for name in names[module_list_idx + 1 :]:
|
||||
# Modules that should be on last rank
|
||||
if not self.pp_group.is_last_rank:
|
||||
# attrsetter in case the module is nested (e.g. "text_model.norm")
|
||||
attrsetter(name)(module, PPMissingLayer())
|
||||
|
||||
def _vocab_embeddings(self) -> set[nn.Embedding]:
|
||||
"""The `nn.Embedding`s in `self.model` which hold a vocab table."""
|
||||
|
||||
def vocab_sizes(config: PreTrainedConfig):
|
||||
for key, value in vars(config).items():
|
||||
if isinstance(value, PreTrainedConfig):
|
||||
yield from vocab_sizes(value)
|
||||
elif "vocab_size" in key and isinstance(value, int):
|
||||
yield value
|
||||
|
||||
sizes = set(vocab_sizes(self.config))
|
||||
# `get_input_embeddings` may return a module which composes the `nn.Embedding`,
|
||||
# or a `PPMissingLayer` if the embeddings are not on this pipeline stage
|
||||
embeddings = {
|
||||
module
|
||||
for module in self.model.get_input_embeddings().modules()
|
||||
if isinstance(module, nn.Embedding)
|
||||
}
|
||||
embeddings.update(
|
||||
module
|
||||
for module in self.model.modules()
|
||||
if isinstance(module, nn.Embedding) and module.num_embeddings in sizes
|
||||
)
|
||||
return embeddings
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Recursively replace modules in the model as needed.
|
||||
|
||||
Currently, this replaces:
|
||||
|
||||
- GLUs with a fused `MergedColumnParallelLinear` + `...AndMul`
|
||||
- Attention QKV projections with a fused `QKVParallelLinear` + split
|
||||
- `nn.Linear` with vLLM's tensor parallel linear classes
|
||||
- `nn.Conv2d` / `nn.Conv3d` with vLLM's `Conv2d` / `Conv3d`
|
||||
- Vocab `nn.Embedding`s with vLLM's `VocabParallelEmbedding`
|
||||
- RMSNorm (detected from their dataflow) with vLLM's `RMSNorm`or `GemmaRMSNorm`
|
||||
- `nn.LayerNorm` with vLLM's `LayerNorm`
|
||||
"""
|
||||
tp_plan = self.model.tp_plan or {}
|
||||
|
||||
if not tp_plan and self.tp_group.world_size > 1:
|
||||
tip = get_feature_request_tip(
|
||||
self.model_config.model, self.model_config.trust_remote_code
|
||||
)
|
||||
logger.warning_once(
|
||||
"%s does not define a tensor parallel plan. The Transformers modeling "
|
||||
"backend will shard the model the best it can during graph fusion and "
|
||||
"replicate the rest. This may be suboptimal or fail if the model does "
|
||||
"not fuse cleanly. %s",
|
||||
type(self.model).__name__,
|
||||
tip,
|
||||
)
|
||||
|
||||
# Prefix the patterns because we always start from `self.model`
|
||||
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
|
||||
# Detect fusable patterns once per module class (cached, so this is cheap)
|
||||
fusers = Fusers(self.model, self.vllm_config)
|
||||
|
||||
vocab_embeddings = self._vocab_embeddings()
|
||||
|
||||
orig_to_new_stacked: dict[str, tuple[str, ShardId]] = {}
|
||||
|
||||
def register_fusion(fuser: BaseFuser, prefix: str, module: nn.Module):
|
||||
"""Register a fused layer's mappings just before it is built."""
|
||||
self.fusers.setdefault(prefix, []).append(fuser)
|
||||
|
||||
if (
|
||||
isinstance(fuser, AttentionFuser)
|
||||
and (index := fuser.layer_index(module)) is not None
|
||||
):
|
||||
if index in self.attention_fusers:
|
||||
other, _ = self.attention_fusers[index]
|
||||
raise ValueError(
|
||||
f"Layer {index} dispatches attention from more than one "
|
||||
f"module ({other} and {prefix}), which one vLLM attention "
|
||||
"layer cannot serve."
|
||||
)
|
||||
self.attention_fusers[index] = (prefix, fuser)
|
||||
|
||||
orig_to_new_stacked.update(fuser.orig_to_new_stacked(prefix))
|
||||
|
||||
packed_modules_mapping = fuser.packed_modules_mapping
|
||||
self.packed_modules_mapping.update(packed_modules_mapping)
|
||||
if self.quant_config is not None:
|
||||
self.quant_config.packed_modules_mapping.update(packed_modules_mapping)
|
||||
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
new_module = child_module
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
if (
|
||||
isinstance(module, nn.ModuleList)
|
||||
and len(module) == self.text_config.num_hidden_layers
|
||||
):
|
||||
# Populate Eagle3 attrs
|
||||
self._target_class = type(child_module)
|
||||
layer_name = qual_name.removeprefix("model.")
|
||||
self._layer_names[int(child_name)] = layer_name
|
||||
# MTP weights should not be loaded into the base model
|
||||
num_hidden_layers = self.text_config.num_hidden_layers
|
||||
names = (
|
||||
"n_predict", # Override from SpeculativeConfig
|
||||
"num_nextn_predict_layers", # Most models
|
||||
"mtp_num_hidden_layers", # Qwen 3.5
|
||||
)
|
||||
n_predict = getattr_iter(self.text_config, names, 0)
|
||||
for i in range(num_hidden_layers, num_hidden_layers + n_predict):
|
||||
mtp_prefix = f"{prefix}.{i}."
|
||||
if mtp_prefix not in self.ignore_unexpected_prefixes:
|
||||
self.ignore_unexpected_prefixes.append(mtp_prefix)
|
||||
# Replace modules as needed
|
||||
if isinstance(child_module, nn.Linear):
|
||||
generator = (p for p in tp_plan if re.match(p, qual_name))
|
||||
pattern = next(generator, None)
|
||||
# Some weight loaders expect all linear layers to inherit
|
||||
# LinearBase, so we set a default style which causes any
|
||||
# unspecified layers to be replaced with ReplicatedLinear
|
||||
style = tp_plan.get(pattern, "replicate")
|
||||
new_module = replace_linear_class(
|
||||
child_module, style, self.quant_config, prefix=qual_name
|
||||
)
|
||||
elif isinstance(child_module, (nn.Conv2d, nn.Conv3d)):
|
||||
new_module = replace_conv_class(child_module)
|
||||
elif child_module in vocab_embeddings:
|
||||
new_module = replace_embedding_class(
|
||||
child_module, self.quant_config, prefix=qual_name
|
||||
)
|
||||
elif isinstance(child_module, nn.LayerNorm):
|
||||
new_module = replace_layernorm_class(child_module)
|
||||
elif child_module_fusers := fusers[child_module]:
|
||||
for fuser in child_module_fusers:
|
||||
register_fusion(fuser, qual_name, child_module)
|
||||
new_module = fuser.fuse(new_module, qual_name, self.vllm_config)
|
||||
logger.info_once(fuser.info(child_name))
|
||||
_recursive_replace(new_module, prefix=qual_name)
|
||||
elif not isinstance(child_module, MoERunner):
|
||||
# MoERunner can contain aliases of shared experts and gates,
|
||||
# so we don't want to recurse into it and break weight loading.
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
if new_module is not child_module:
|
||||
setattr(module, child_name, new_module)
|
||||
log_replacement(qual_name, child_module, new_module)
|
||||
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(orig_to_new_stacked=orig_to_new_stacked)
|
||||
|
||||
def _create_attention_instances(self, prefix: str = ""):
|
||||
"""Create `Attention` instances to inform KV cache allocation."""
|
||||
text_config = self.text_config
|
||||
attn_cls = self._get_attn_cls()
|
||||
|
||||
# kv_lora_rank indicates that this is an MLA model
|
||||
if getattr(text_config, "kv_lora_rank", None) is not None:
|
||||
if attn_cls is MLAAttention:
|
||||
text_config._attn_implementation = VLLM_MLA_ATTN_IMPL
|
||||
else:
|
||||
# MLA model not using MLAAttention: recompute head_size for full attn
|
||||
qk_nope_head_dim = getattr(text_config, "qk_nope_head_dim", 0)
|
||||
qk_rope_head_dim = getattr(text_config, "qk_rope_head_dim", 0)
|
||||
if qk_head_dim := qk_nope_head_dim + qk_rope_head_dim:
|
||||
self.model_config.model_arch_config.head_size = qk_head_dim
|
||||
|
||||
logits_soft_cap = getattr(text_config, "attn_logit_softcapping", None)
|
||||
layer_types = getattr(text_config, "layer_types", None)
|
||||
|
||||
pp_rank = self.pp_group.rank_in_group
|
||||
pp_size = self.pp_group.world_size
|
||||
start, end = get_pp_indices(text_config.num_hidden_layers, pp_rank, pp_size)
|
||||
|
||||
for i in range(start, end):
|
||||
if i not in self.attention_fusers:
|
||||
layer = (
|
||||
f"{i} ({layer_types[i]})"
|
||||
if layer_types and i < len(layer_types)
|
||||
else str(i)
|
||||
)
|
||||
raise ValueError(
|
||||
f"Layer {layer} does not dispatch through the Transformers "
|
||||
"attention interface and vLLM has no other way to handle it."
|
||||
)
|
||||
attn_prefix, attn_fuser = self.attention_fusers[i]
|
||||
attn_module = self.get_submodule(attn_prefix)
|
||||
|
||||
# `[i]` is the whole-model config unless the checkpoint is
|
||||
# heterogeneous, in which case it is this layer's own geometry.
|
||||
arch_config = self.model_config.model_arch_config[i]
|
||||
num_heads = self.model_config.get_num_attention_heads(
|
||||
self.parallel_config, arch_config
|
||||
)
|
||||
head_size = arch_config.head_size
|
||||
if (scale := attn_fuser.scale(attn_module)) is None:
|
||||
# Default to Llama scale if AttentionFuser couldn't identify it
|
||||
scale = head_size**-0.5
|
||||
|
||||
kwargs = dict(
|
||||
num_heads=num_heads,
|
||||
scale=scale,
|
||||
cache_config=self.cache_config,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, f"{attn_prefix}.{VLLM_ATTN_ATTR}"),
|
||||
)
|
||||
|
||||
if attn_cls is MLAAttention:
|
||||
mla_fuser = next(
|
||||
(f for f in self.fusers[attn_prefix] if isinstance(f, MLAFuser)),
|
||||
None,
|
||||
)
|
||||
if mla_fuser is None:
|
||||
raise ValueError(
|
||||
f"Layer {i} ({attn_prefix}) did not fuse as MLA, but the rest "
|
||||
"of the model did, so its latent KV cache cannot be allocated."
|
||||
)
|
||||
from vllm.model_executor.layers.attention.mla_attention import (
|
||||
get_mla_dims,
|
||||
)
|
||||
|
||||
dims = get_mla_dims(self.model_config)
|
||||
kwargs.update(
|
||||
qk_nope_head_dim=dims.qk_nope_head_dim,
|
||||
qk_rope_head_dim=dims.qk_rope_head_dim,
|
||||
v_head_dim=dims.v_head_dim,
|
||||
q_lora_rank=dims.q_lora_rank,
|
||||
kv_lora_rank=dims.kv_lora_rank,
|
||||
kv_b_proj=attn_module.get_submodule(mla_fuser.kv_b_proj_name),
|
||||
)
|
||||
else:
|
||||
kwargs.update(
|
||||
head_size=head_size,
|
||||
num_kv_heads=self.model_config.get_num_kv_heads(
|
||||
self.parallel_config, arch_config
|
||||
),
|
||||
logits_soft_cap=logits_soft_cap,
|
||||
)
|
||||
|
||||
# Handle interleaved sliding window attention
|
||||
if layer_types and layer_types[i] == "sliding_attention":
|
||||
kwargs["per_layer_sliding_window"] = text_config.sliding_window
|
||||
# Handle attention sinks
|
||||
if (sinks := attn_fuser.sinks(attn_module)) is not None:
|
||||
kwargs["sinks"] = sinks
|
||||
|
||||
attn_instance = attn_cls(**kwargs)
|
||||
|
||||
# Attach the instance to the module it serves, which is the only way it
|
||||
# is in the module tree at all: a plain dict is invisible to
|
||||
# `nn.Module.__setattr__`. Being in the tree runs its
|
||||
# `process_weights_after_loading` and moves its KV-scale buffers with
|
||||
# `model.to(device)`, and lets the attention interface read it off the
|
||||
# module rather than index by `module.layer_idx`, an `int` that bakes
|
||||
# layer identity into the traced graph and so costs one compiled
|
||||
# artifact per layer.
|
||||
setattr(attn_module, VLLM_ATTN_ATTR, attn_instance)
|
||||
|
||||
def _get_attn_cls(self) -> type[AttentionLayerBase]:
|
||||
"""Return the `Attention` class to use for this model's layers."""
|
||||
# In encoder models, the attention layers will have `is_causal=False`
|
||||
is_encoder = lambda module: not getattr(module, "is_causal", True)
|
||||
has_encoder = lambda model: any(is_encoder(m) for m in model.modules())
|
||||
is_multimodal = lambda config: config != config.get_text_config()
|
||||
# vLLM does not support encoder-decoder models, so if any encoder layer is
|
||||
# found in a text only model, we assume the whole model is an encoder model
|
||||
if has_encoder(self.model) and not is_multimodal(self.config):
|
||||
return EncoderOnlyAttention
|
||||
if self.model_config.use_mla:
|
||||
if any(
|
||||
isinstance(fuser, MLAFuser)
|
||||
for fusers in self.fusers.values()
|
||||
for fuser in fusers
|
||||
):
|
||||
return MLAAttention
|
||||
logger.warning_once(
|
||||
"This model uses MLA but `MLAFuser` failed to match and/or fuse any "
|
||||
"MLA attention layers. Falling back to full attention with a padded "
|
||||
"`value` head dimension."
|
||||
)
|
||||
os.environ["VLLM_MLA_DISABLE"] = "1"
|
||||
return Attention
|
||||
|
||||
def init_parameters(self, module: nn.Module, dtype: torch.dtype | None = None):
|
||||
"""If a `parameter` is on the `meta` device, then its parent
|
||||
`module` is the original module created by:
|
||||
|
||||
```python
|
||||
with torch.device("meta"):
|
||||
self.model: "PreTrainedModel" = AutoModel.from_config(...)
|
||||
```
|
||||
"""
|
||||
dtype = dtype or self.model_config.dtype
|
||||
device = self.device_config.device
|
||||
|
||||
def _init_parameters(module: nn.Module):
|
||||
for name, param in module.named_parameters(recurse=False):
|
||||
# Already on device, nothing to do
|
||||
if param.device != torch.device("meta"):
|
||||
continue
|
||||
# Already a vLLM parameter, nothing to do
|
||||
if hasattr(param, "weight_loader"):
|
||||
continue
|
||||
data = torch.empty_like(param.data, dtype=dtype, device=device)
|
||||
setattr(module, name, nn.Parameter(data=data))
|
||||
for child in module.children():
|
||||
_init_parameters(child)
|
||||
|
||||
_init_parameters(module)
|
||||
|
||||
def keep_in_fp32(self, module: nn.Module):
|
||||
"""Honor `_keep_in_fp32_modules_strict` as `from_pretrained` would."""
|
||||
if self.model_config.dtype not in (torch.float16, torch.bfloat16):
|
||||
return
|
||||
fragments = getattr(module, "_keep_in_fp32_modules_strict", None)
|
||||
if not fragments:
|
||||
return
|
||||
pattern = re.compile("|".join(re.escape(f) for f in fragments))
|
||||
for name, tensor in named_state(module):
|
||||
if not hasattr(tensor, "weight_loader") and pattern.search(name):
|
||||
tensor.data = tensor.data.to(torch.float32)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.get_input_embeddings()(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if not self.pp_group.is_first_rank:
|
||||
assert intermediate_tensors is not None
|
||||
input_ids = None
|
||||
inputs_embeds = intermediate_tensors["hidden_states"]
|
||||
|
||||
# Add batch dimension before entering Transformers model
|
||||
if input_ids is not None and input_ids.ndim == 1:
|
||||
# [seq_len] -> [1, seq_len]
|
||||
input_ids = input_ids[None, ...]
|
||||
if inputs_embeds is not None and inputs_embeds.ndim == 2:
|
||||
# [seq_len, hidden_size] -> [1, seq_len, hidden_size]
|
||||
inputs_embeds = inputs_embeds[None, ...]
|
||||
if positions.ndim == 1:
|
||||
# [seq_len] -> [1, seq_len]
|
||||
positions = positions[None, ...]
|
||||
|
||||
# Transformers models expect either input_ids or inputs_embeds, but not both
|
||||
if input_ids is not None and inputs_embeds is not None:
|
||||
input_ids = None
|
||||
|
||||
outputs = self.model(
|
||||
input_ids=input_ids,
|
||||
inputs_embeds=inputs_embeds,
|
||||
use_cache=False,
|
||||
position_ids=positions,
|
||||
return_dict=False,
|
||||
**self._output_aux_hidden_states_kwargs,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Remove batch dimension after exiting Transformers model
|
||||
hidden_states = outputs[0][0, ...]
|
||||
if self._output_aux_hidden_states_kwargs:
|
||||
aux_hidden_states = [x[0][0, ...] for x in outputs[1:]]
|
||||
|
||||
if not self.pp_group.is_last_rank:
|
||||
return IntermediateTensors({"hidden_states": hidden_states})
|
||||
|
||||
if self._output_aux_hidden_states_kwargs and len(aux_hidden_states) > 0:
|
||||
return hidden_states, aux_hidden_states
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
ignore_unexpected_prefixes=self.ignore_unexpected_prefixes,
|
||||
ignore_unexpected_suffixes=self.ignore_unexpected_suffixes,
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
@staticmethod
|
||||
def check_version(min_version: str, feature: str):
|
||||
installed = Version(transformers.__version__)
|
||||
required = Version(min_version)
|
||||
if installed < required:
|
||||
raise ImportError(
|
||||
f"Transformers modeling backend requires transformers>={required} "
|
||||
f"for {feature}, but got {installed}"
|
||||
)
|
||||
|
||||
def set_aux_hidden_state_layers(self, layers: tuple[int, ...]) -> None:
|
||||
from transformers.utils.output_capturing import (
|
||||
OutputRecorder,
|
||||
maybe_install_capturing_hooks,
|
||||
)
|
||||
|
||||
# The default value in PreTrainedModel is None
|
||||
if self.model._can_record_outputs is None:
|
||||
self.model._can_record_outputs = {}
|
||||
|
||||
target_class = self._target_class
|
||||
for layer in layers:
|
||||
# layer - 1 because we want the input to the layer
|
||||
layer_name = self._layer_names[layer - 1]
|
||||
layer_key = f"aux_hidden_state_{layer}"
|
||||
aux_hidden_state_i = OutputRecorder(target_class, layer_name=layer_name)
|
||||
self.model._can_record_outputs[layer_key] = aux_hidden_state_i
|
||||
self._output_aux_hidden_states_kwargs[f"output_{layer_key}"] = True
|
||||
|
||||
# Ensure that the capture hooks are installed before dynamo traces the model
|
||||
maybe_install_capturing_hooks(self.model)
|
||||
|
||||
def get_eagle3_default_aux_hidden_state_layers(self) -> tuple[int, ...]:
|
||||
num_layers = self.text_config.num_hidden_layers
|
||||
return (2, num_layers // 2, num_layers - 3)
|
||||
@@ -1,89 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for causal language models."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
ParallelLMHead,
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces_base import VllmModelForTextGeneration
|
||||
from vllm.model_executor.models.utils import PPMissingLayer, maybe_prefix
|
||||
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class CausalMixin(VllmModelForTextGeneration, Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip VllmModelForTextGeneration.__init__ and call the next class in MRO
|
||||
super(VllmModelForTextGeneration, self).__init__(
|
||||
vllm_config=vllm_config, prefix=prefix
|
||||
)
|
||||
|
||||
tie_word_embeddings = self._get_tie_word_embeddings()
|
||||
|
||||
if self.pp_group.is_last_rank:
|
||||
self.lm_head = ParallelLMHead(
|
||||
self.text_config.vocab_size,
|
||||
self.text_config.hidden_size,
|
||||
quant_config=self.quant_config,
|
||||
prefix=maybe_prefix(prefix, "lm_head"),
|
||||
)
|
||||
if tie_word_embeddings:
|
||||
for module in self.model.get_input_embeddings().modules():
|
||||
if isinstance(module, VocabParallelEmbedding):
|
||||
self.lm_head = self.lm_head.tie_weights(module)
|
||||
break
|
||||
|
||||
self.logits_processor = LogitsProcessor(
|
||||
self.text_config.vocab_size,
|
||||
scale=getattr(self.text_config, "logit_scale", 1.0),
|
||||
soft_cap=getattr(self.text_config, "final_logit_softcapping", None),
|
||||
)
|
||||
else:
|
||||
self.lm_head = PPMissingLayer()
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, "torch.Tensor"]]) -> set[str]:
|
||||
"""A thin wrapper around `Base.load_weights` to handle the lm_head bias."""
|
||||
lm_head_bias = set()
|
||||
load_device = self.vllm_config.load_config.device or self.device_config.device
|
||||
|
||||
def auto_load_lm_head_bias(weights):
|
||||
for name, weight in weights:
|
||||
if name.endswith("lm_head.bias") and self.pp_group.is_last_rank:
|
||||
# load_weights is called outside vLLM's device context,
|
||||
# so we must enter it again when registering the bias.
|
||||
with torch.device(load_device):
|
||||
self.lm_head._register_bias()
|
||||
self.lm_head.bias.weight_loader(self.lm_head.bias, weight)
|
||||
lm_head_bias.add(name)
|
||||
else:
|
||||
yield name, weight
|
||||
|
||||
return super().load_weights(auto_load_lm_head_bias(weights)) | lm_head_bias
|
||||
|
||||
def compute_logits(self, hidden_states: "torch.Tensor") -> "torch.Tensor | None":
|
||||
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
|
||||
return logits
|
||||
@@ -1,126 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fuser detection for the Transformers modeling backend."""
|
||||
|
||||
from collections import UserDict
|
||||
from typing import TYPE_CHECKING, TypeVar
|
||||
|
||||
from cachetools import cached
|
||||
from torch import nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers import (
|
||||
AttentionFuser,
|
||||
BaseFuser,
|
||||
GLUFuser,
|
||||
MergedColumnParallelFuser,
|
||||
MLAFuser,
|
||||
PackedQKVFuser,
|
||||
QKVFuser,
|
||||
RewriteFuser,
|
||||
RMSNormFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import trace
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
F = TypeVar("F", bound=BaseFuser)
|
||||
|
||||
|
||||
def key(module: nn.Module) -> tuple:
|
||||
"""Cache key for `get_fusers`. Considers module type and its immediate children."""
|
||||
return (type(module), tuple(name for name, _ in module.named_children()))
|
||||
|
||||
|
||||
FUSERS: tuple[type[BaseFuser], ...] = (
|
||||
# Order these by priority, e.g. QKV would be preferred over MergedColumnParallel
|
||||
MLAFuser,
|
||||
GLUFuser,
|
||||
QKVFuser,
|
||||
PackedQKVFuser,
|
||||
MergedColumnParallelFuser,
|
||||
RMSNormFuser,
|
||||
# Put fusers that don't redefine forward after this comment
|
||||
AttentionFuser,
|
||||
)
|
||||
"""Every fuser, in priority order: those that redefine the forward first, then those
|
||||
that leave it alone. A new fuser is added here."""
|
||||
|
||||
|
||||
@cached(cache={}, key=key)
|
||||
def get_fusers(module: nn.Module) -> list[BaseFuser]:
|
||||
"""Every fuser that could apply to `module`'s class (cached), in `FUSERS` order."""
|
||||
# Projection fusions need >=2 sibling linears; the RMSNorm fusion needs a
|
||||
# leaf module (raw tensor math, no submodules). Nothing else can match, and
|
||||
# tracing is skipped for it.
|
||||
n_linear = sum(isinstance(c, nn.Linear) for c in module.children())
|
||||
is_leaf = next(module.children(), None) is None
|
||||
graph = trace(module) if n_linear >= 2 or is_leaf else None
|
||||
|
||||
# Every redefining candidate that matches is cached, not just the first:
|
||||
# instances of this class can be shaped heterogeneously, so which one is
|
||||
# usable is decided per instance in Fusers.__getitem__, not here. A forward
|
||||
# can only be rewritten from a trace of it, so each starts fresh regardless.
|
||||
fusers: list[BaseFuser] = []
|
||||
for fuser_cls in FUSERS:
|
||||
if fuser_cls.redefines_forward and graph is None:
|
||||
continue
|
||||
if (fuser := fuser_cls.match(graph, module)) is None:
|
||||
continue
|
||||
if isinstance(fuser, RewriteFuser):
|
||||
try:
|
||||
fuser.update_forward(module)
|
||||
except Exception as exc:
|
||||
logger.debug(
|
||||
"Attempted to fuse %s using %s but failed "
|
||||
"to update its forward method: %s",
|
||||
type(module),
|
||||
fuser_cls.__name__,
|
||||
exc,
|
||||
)
|
||||
continue
|
||||
fusers.append(fuser)
|
||||
|
||||
# A norm we could not match structurally is left unfused; flag likely misses.
|
||||
if (
|
||||
module.__class__.__name__.endswith("RMSNorm")
|
||||
and graph is not None
|
||||
and not any(isinstance(fuser, RMSNormFuser) for fuser in fusers)
|
||||
):
|
||||
logger.warning_once(
|
||||
"%s looks like an RMSNorm but its computation did not match the "
|
||||
"expected pattern, so it was left unfused.",
|
||||
module.__class__.__name__,
|
||||
)
|
||||
return fusers
|
||||
|
||||
|
||||
def get_fuser(module: nn.Module, fuser_cls: type[F]) -> "F | None":
|
||||
"""The first `fuser_cls` that applies to `module`, if one does."""
|
||||
return next((f for f in get_fusers(module) if isinstance(f, fuser_cls)), None)
|
||||
|
||||
|
||||
class Fusers(UserDict):
|
||||
"""Mapping from module class and shape to fusers, for all fusable modules."""
|
||||
|
||||
def __init__(self, model: nn.Module, vllm_config: "VllmConfig"):
|
||||
self.vllm_config = vllm_config
|
||||
super().__init__({key(m): get_fusers(m) for m in model.modules()})
|
||||
|
||||
def __getitem__(self, m: nn.Module) -> list[BaseFuser]:
|
||||
"""The fusers this instance can take, in the order to apply them."""
|
||||
chosen: list[BaseFuser] = []
|
||||
redefined = False
|
||||
for fuser in self.data.get(key(m), ()):
|
||||
# Only one candidate may rewrite m's forward; take the first
|
||||
# (highest-priority) one that validates for this instance.
|
||||
if fuser.redefines_forward and redefined:
|
||||
continue
|
||||
if not fuser.validate(m, self.vllm_config):
|
||||
continue
|
||||
redefined = redefined or fuser.redefines_forward
|
||||
chosen.append(fuser)
|
||||
return chosen
|
||||
@@ -1,33 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Concrete fusers for the Transformers modeling backend."""
|
||||
|
||||
from vllm.model_executor.models.transformers.fusers.attention import AttentionFuser
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
BaseFuser,
|
||||
RewriteFuser,
|
||||
StackedFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.merged_column import (
|
||||
MergedColumnParallelFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fusers.mla import MLAFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.transformers.fusers.packed_qkv import PackedQKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers.qkv import QKVFuser
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
|
||||
|
||||
__all__ = [
|
||||
"AttentionFuser",
|
||||
"BaseFuser",
|
||||
"RewriteFuser",
|
||||
"StackedFuser",
|
||||
"GLUFuser",
|
||||
"MergedColumnParallelFuser",
|
||||
"MLAFuser",
|
||||
"MoEBlockFuser",
|
||||
"PackedQKVFuser",
|
||||
"QKVFuser",
|
||||
"RMSNormFuser",
|
||||
]
|
||||
@@ -1,162 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Attention fuser: the module that dispatches to the attention interface."""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass
|
||||
from functools import cache
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.model_executor.model_loader.weight_utils import sharded_weight_loader
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.utils import set_weight_attrs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
VLLM_ATTN_IMPL = "vllm"
|
||||
VLLM_MLA_ATTN_IMPL = "vllm_mla"
|
||||
|
||||
|
||||
def _is_interface_lookup(node: ast.expr | None) -> bool:
|
||||
"""Whether `node` reads an entry out of `ALL_ATTENTION_FUNCTIONS`."""
|
||||
# ALL_ATTENTION_FUNCTIONS.get_interface(...) or ALL_ATTENTION_FUNCTIONS[...]
|
||||
if isinstance(node, ast.Call):
|
||||
node = node.func
|
||||
if not isinstance(node, (ast.Attribute, ast.Subscript)):
|
||||
return False
|
||||
return (
|
||||
isinstance(node.value, ast.Name) and node.value.id == "ALL_ATTENTION_FUNCTIONS"
|
||||
)
|
||||
|
||||
|
||||
@cache
|
||||
def interface_call(forward: Callable) -> ast.Call | None:
|
||||
"""The attention interface call in `forward`, if it makes exactly one."""
|
||||
try:
|
||||
source = inspect.getsource(inspect.unwrap(forward))
|
||||
tree = ast.parse(textwrap.dedent(source))
|
||||
except (AttributeError, OSError, SyntaxError, TypeError):
|
||||
return None
|
||||
|
||||
# Find the names of the local variables that read from the interface lookup
|
||||
names: set[str] = set()
|
||||
for node in ast.walk(tree):
|
||||
if not isinstance(node, (ast.Assign, ast.AnnAssign)):
|
||||
continue
|
||||
if not _is_interface_lookup(node.value):
|
||||
continue
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
names.update(t.id for t in targets if isinstance(t, ast.Name))
|
||||
# Find the calls to those names, and return the one if there is exactly one
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(tree)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Name)
|
||||
and node.func.id in names
|
||||
]
|
||||
return calls[0] if len(calls) == 1 else None
|
||||
|
||||
|
||||
def _resolve(node: ast.expr, module: nn.Module) -> object:
|
||||
"""The value of `node` on `module`, for literals and `self.<attr>`."""
|
||||
if isinstance(node, ast.Constant):
|
||||
return node.value
|
||||
if (
|
||||
isinstance(node, ast.Attribute)
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
):
|
||||
return getattr(module, node.attr, None)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class AttentionFuser(BaseFuser):
|
||||
"""A module that dispatches through the Transformers attention interface."""
|
||||
|
||||
redefines_forward: ClassVar[bool] = False
|
||||
|
||||
source_cls: str
|
||||
"""Class of the HF module that dispatches (for logging)."""
|
||||
scale_expr: ast.expr | None = None
|
||||
"""Source of the `scaling=` the module hands the interface, if it hands one."""
|
||||
s_aux_expr: ast.expr | None = None
|
||||
"""Source of the `s_aux=` the module hands the interface, if it hands one."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
return f"Found: {name} ({self.source_cls}) -> attention interface"
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls, graph: fx.Graph | None, module: nn.Module
|
||||
) -> "AttentionFuser | None":
|
||||
if (call := interface_call(type(module).forward)) is None:
|
||||
return None
|
||||
scaling = [kw.value for kw in call.keywords if kw.arg == "scaling"]
|
||||
scale_expr = scaling[0] if len(scaling) == 1 else None
|
||||
s_aux = [kw.value for kw in call.keywords if kw.arg == "s_aux"]
|
||||
s_aux_expr = s_aux[0] if len(s_aux) == 1 else None
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
scale_expr=scale_expr,
|
||||
s_aux_expr=s_aux_expr,
|
||||
)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether `module` will actually dispatch to vLLM."""
|
||||
config = getattr(module, "config", None)
|
||||
# Only patched in the text config, this excludes attention based mm encoders
|
||||
vllm_attn_impls = {VLLM_ATTN_IMPL, VLLM_MLA_ATTN_IMPL}
|
||||
return getattr(config, "_attn_implementation", None) in vllm_attn_impls
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
if (sinks := self.sinks(module)) is not None:
|
||||
assert isinstance(self.s_aux_expr, ast.Attribute)
|
||||
size = sinks.numel() // vllm_config.parallel_config.tensor_parallel_size
|
||||
device = vllm_config.device_config.device
|
||||
data = torch.empty(size, dtype=sinks.dtype, device=device)
|
||||
sinks_param = nn.Parameter(data, requires_grad=False)
|
||||
set_weight_attrs(sinks_param, {"weight_loader": sharded_weight_loader(0)})
|
||||
setattr(module, self.s_aux_expr.attr, sinks_param)
|
||||
return module
|
||||
|
||||
def layer_index(self, module: nn.Module) -> int | None:
|
||||
"""The layer `module` computes attention for, if it declares one."""
|
||||
layer_idx = getattr(module, "layer_idx", None)
|
||||
return layer_idx if isinstance(layer_idx, int) else None
|
||||
|
||||
def scale(self, module: nn.Module) -> float | None:
|
||||
"""The softmax scale `module` passes to the interface, or `None`."""
|
||||
if self.scale_expr is None:
|
||||
return None
|
||||
scale = _resolve(self.scale_expr, module)
|
||||
if not isinstance(scale, (int, float)) or isinstance(scale, bool):
|
||||
expression = ast.unparse(self.scale_expr)
|
||||
raise ValueError(
|
||||
f"Cannot resolve attention scaling expression {expression!r} in "
|
||||
f"{type(module).__name__}."
|
||||
)
|
||||
return float(scale)
|
||||
|
||||
def sinks(self, module: nn.Module) -> nn.Parameter | None:
|
||||
"""The per-head sink tensor `module` passes to the interface, or `None`."""
|
||||
if self.s_aux_expr is None:
|
||||
return None
|
||||
s_aux = _resolve(self.s_aux_expr, module)
|
||||
if not isinstance(s_aux, (nn.Parameter, type(None))):
|
||||
expression = ast.unparse(self.s_aux_expr)
|
||||
raise ValueError(
|
||||
f"Cannot resolve attention s_aux expression {expression!r} in "
|
||||
f"{type(module).__name__}."
|
||||
)
|
||||
return s_aux
|
||||
@@ -1,250 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Base classes for the Transformers backend fusers."""
|
||||
|
||||
import ast
|
||||
import types
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Callable, Iterable
|
||||
from dataclasses import dataclass, field
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
aliasing_names,
|
||||
bypass_existence_guard,
|
||||
replace_expr,
|
||||
self_call_and_refs,
|
||||
written_names,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
@dataclass
|
||||
class BaseFuser(ABC):
|
||||
"""A detected fusion and how to apply it.
|
||||
|
||||
`match` analyses the module *class* once (cached, see `get_fusers`); `fuse`
|
||||
then applies the fusion to an instance in `recursive_replace`, returning the
|
||||
module to install in its place.
|
||||
"""
|
||||
|
||||
redefines_forward: ClassVar[bool] = True
|
||||
"""Whether `fuse` gives the module a different forward,
|
||||
by rewriting its source or by returning a different module in its place."""
|
||||
|
||||
@abstractmethod
|
||||
def info(self, name: str) -> str:
|
||||
"""A human-readable description of the fusion at `name`, for logging."""
|
||||
|
||||
@classmethod
|
||||
@abstractmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "BaseFuser | None":
|
||||
"""Match the pattern in `graph`, returning a fuser if found."""
|
||||
|
||||
@abstractmethod
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether this fuser can be applied to this `module` instance."""
|
||||
|
||||
@abstractmethod
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Apply the fusion to an already-validated `module`, returning the
|
||||
module to install in its place (mutated in place, or freshly built)."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries this fuser contributes
|
||||
(none unless it stacks weights)."""
|
||||
return {}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`packed_modules_mapping` entries this fuser contributes (none unless
|
||||
it stacks weights)."""
|
||||
return {}
|
||||
|
||||
|
||||
def fused_head_size(module: nn.Module, vllm_config: "VllmConfig") -> int:
|
||||
"""The head size of `module`, which head counts are derived from.
|
||||
|
||||
Prefer the module's own `head_dim`, which Transformers sets per instance:
|
||||
the model-wide value is the largest head size across layers, so on a
|
||||
heterogeneous checkpoint it would divide out the wrong number of heads, and
|
||||
it is the text head size, so it does not describe a vision tower at all.
|
||||
"""
|
||||
return getattr(module, "head_dim", None) or vllm_config.model_config.get_head_size()
|
||||
|
||||
|
||||
def local_output_sizes(merged_name: str) -> str:
|
||||
"""Source for the per-rank widths of the merged linear `self.<merged_name>`."""
|
||||
merged = f"self.{merged_name}"
|
||||
return f"[s // {merged}.tp_size for s in {merged}.output_sizes]"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RewriteFuser(BaseFuser):
|
||||
"""A fuser that rewrites the module's forward and rebinds it.
|
||||
|
||||
`match` and `update_forward` analyse the class once; `fuse` swaps the
|
||||
submodules and binds the compiled forward on an instance in place, so it
|
||||
keeps its class and any attribute the fusion does not consume.
|
||||
"""
|
||||
|
||||
source_cls: str
|
||||
"""Class of the HF module the fused projections belonged to (for logging)."""
|
||||
|
||||
fused_forward: Callable = field(init=False, repr=False)
|
||||
"""The compiled rewritten forward, set by `update_forward`."""
|
||||
|
||||
@abstractmethod
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite and compile `type(module)`'s forward source.
|
||||
|
||||
Raises if the source does not admit the rewrite (fusion is then skipped).
|
||||
"""
|
||||
|
||||
@abstractmethod
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace `module`'s submodules with their vLLM equivalents."""
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse an already-validated `module` in place (see `Fusers.__getitem__`).
|
||||
|
||||
Builds the merged submodule and binds the compiled forward."""
|
||||
self.update_attrs(module, prefix, vllm_config)
|
||||
module.forward = types.MethodType(self.fused_forward, module)
|
||||
return module
|
||||
|
||||
|
||||
@dataclass
|
||||
class StackedFuser(RewriteFuser):
|
||||
"""A fuser that merges sibling projections into one stacked linear and
|
||||
rewrites the forward to call it."""
|
||||
|
||||
merged_cls_name: ClassVar[str]
|
||||
"""Name of the vLLM class the merged projection becomes (for logging)."""
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def merged_name(self) -> str:
|
||||
"""Attribute name of the merged module created by `update_attrs`."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
sources = " + ".join(shard for shard, _ in self.shards)
|
||||
return (
|
||||
f"Fused: {sources} ({name}: {self.source_cls}) -> "
|
||||
f"{self.merged_name} ({self.merged_cls_name})"
|
||||
)
|
||||
|
||||
@property
|
||||
@abstractmethod
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""Each projection's original name and its shard id in the merged module.
|
||||
|
||||
Source for both `orig_to_new_stacked` and `packed_modules_mapping`."""
|
||||
|
||||
def orig_to_new_stacked(self, prefix: str) -> dict[str, tuple[str, ShardId]]:
|
||||
"""`WeightsMapper.orig_to_new_stacked` entries for one fused instance.
|
||||
|
||||
Maps each checkpoint name to `(merged_name, shard_id)`, keyed by qualname
|
||||
so only this exact layer is remapped, never a same-named projection
|
||||
elsewhere (e.g. an unfused MoE expert's `gate_proj`)."""
|
||||
merged = maybe_prefix(prefix, self.merged_name)
|
||||
return {
|
||||
maybe_prefix(prefix, name): (merged, shard) for name, shard in self.shards
|
||||
}
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
"""`{merged_name: [projection names]}` so quantization can unpack the
|
||||
fused layer into its per-shard configs."""
|
||||
return {self.merged_name: [name for name, _ in self.shards]}
|
||||
|
||||
def _unguarded_calls(
|
||||
self, funcdef: ast.FunctionDef, names: Iterable[str]
|
||||
) -> list[ast.Call]:
|
||||
"""One `self.<name>(arg)` call per projection, existence guards folded.
|
||||
|
||||
`update_attrs` deletes the projections it merges, so any reference to one beyond
|
||||
its call site must be a guard on its existence; folding those to their constant
|
||||
value keeps the rewritten forward off a name that no longer exists. A reference
|
||||
that is not such a guard raises, so fusion is skipped."""
|
||||
calls = []
|
||||
for name in names:
|
||||
call, refs = self_call_and_refs(funcdef, name)
|
||||
for ref in refs:
|
||||
bypass_existence_guard(funcdef, ref, name)
|
||||
calls.append(call)
|
||||
return calls
|
||||
|
||||
def _check_input_stable(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
module: nn.Module,
|
||||
calls: list[ast.Call],
|
||||
block: list[ast.stmt],
|
||||
indices: list[int],
|
||||
) -> None:
|
||||
"""Raise unless hoisting the merged GEMM preserves what it reads.
|
||||
|
||||
Fusing moves every projection to one call at `min(indices)`, so the
|
||||
merged GEMM reads the input once, up front, where the last of `calls`
|
||||
would have read it later. That holds only if nothing in between changes
|
||||
the input, and a change need not name it: writing any view that shares
|
||||
its storage changes it too. Both halves of the check over-approximate,
|
||||
since a false hit costs a fusion while a miss returns wrong numbers.
|
||||
"""
|
||||
arg_names = {
|
||||
node.id
|
||||
for node in ast.walk(calls[0].args[0])
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
|
||||
}
|
||||
# Protect the names the input reads and anything that may alias them.
|
||||
tracked = aliasing_names(funcdef, arg_names, module)
|
||||
# Any of them rebound, written through, or passed to a call that writes
|
||||
# its argument would leave the hoisted GEMM reading a different value.
|
||||
region = block[min(indices) : max(indices) + 1]
|
||||
if tracked & written_names(region):
|
||||
raise ValueError("projection input is rebound or mutated before all calls")
|
||||
|
||||
def _splice_merged_split(
|
||||
self,
|
||||
funcdef: ast.FunctionDef,
|
||||
calls: list[ast.Call],
|
||||
block: list[ast.stmt],
|
||||
index: int,
|
||||
) -> None:
|
||||
"""Insert `temps = self.<merged_name>(arg).split(sizes, -1)` at
|
||||
`block[index]` and replace each of `calls` with its temp name.
|
||||
|
||||
`calls` must share one input argument; `block[index]` must be where
|
||||
they are (or would be) evaluated. Raises if a generated temporary
|
||||
would shadow an existing name in `funcdef`.
|
||||
"""
|
||||
temps = [f"_vllm_merged_{i}" for i in range(len(calls))]
|
||||
names = {node.id for node in ast.walk(funcdef) if isinstance(node, ast.Name)}
|
||||
if names & set(temps):
|
||||
raise ValueError("fused temporaries would shadow existing names")
|
||||
targets = ", ".join(temps)
|
||||
sections = local_output_sizes(self.merged_name)
|
||||
source = f"{targets} = self.{self.merged_name}(__arg__).split({sections}, -1)"
|
||||
assign = ast.parse(source).body[0]
|
||||
arg = next(
|
||||
node
|
||||
for node in ast.walk(assign)
|
||||
if isinstance(node, ast.Name) and node.id == "__arg__"
|
||||
)
|
||||
replace_expr(assign, arg, calls[0].args[0])
|
||||
ast.copy_location(assign, block[index])
|
||||
block.insert(index, assign)
|
||||
for call, temp in zip(calls, temps):
|
||||
replace_expr(funcdef, call, ast.Name(id=temp, ctx=ast.Load()))
|
||||
@@ -1,191 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GLU projection fuser: `act(gate(x)) * up(x)` -> a fused gate/up linear."""
|
||||
|
||||
import ast
|
||||
import operator
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
from transformers.activations import ACT2CLS
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_AND_MUL_REGISTRY
|
||||
from vllm.model_executor.models.transformers.fusers.merged_column import (
|
||||
MergedColumnParallelFuser,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
find_node,
|
||||
is_linear,
|
||||
peel,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
single_self_call,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import get_act_and_mul_fn
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
CLS2ACT: dict[type, list[str]] = {}
|
||||
for _act_name, _act_cls in ACT2CLS.items():
|
||||
if isinstance(_act_cls, tuple):
|
||||
_act_cls = _act_cls[0]
|
||||
CLS2ACT.setdefault(_act_cls, []).append(_act_name)
|
||||
|
||||
ACT_AND_MUL_NAMES = frozenset(_ACTIVATION_AND_MUL_REGISTRY.keys())
|
||||
|
||||
|
||||
@dataclass
|
||||
class GLUFuser(MergedColumnParallelFuser):
|
||||
"""Fuser for the GLU pattern `act(gate(x)) * up(x)`."""
|
||||
|
||||
act_name: str
|
||||
down_name: str | None
|
||||
merged_name: ClassVar[str] = "gate_up_proj"
|
||||
|
||||
@property
|
||||
def gate_name(self) -> str:
|
||||
return self.linear_names[0]
|
||||
|
||||
@property
|
||||
def up_name(self) -> str:
|
||||
return self.linear_names[1]
|
||||
|
||||
@classmethod
|
||||
def _is_act_of_gate(cls, node: fx.Node, module: nn.Module) -> bool:
|
||||
"""Is node `act(gate(x))` where `gate` is linear and `act` is not linear."""
|
||||
return (
|
||||
node.op == "call_module"
|
||||
and not is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
and is_linear(node.args[0], module)
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _get_glu_nodes(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Search graph for the GLU pattern `act(gate(x)) * up(x)`."""
|
||||
for mul in graph.nodes:
|
||||
if (
|
||||
mul.op == "call_function"
|
||||
and mul.target == operator.mul
|
||||
and len(mul.args) == 2
|
||||
and all(isinstance(arg, fx.Node) for arg in mul.args)
|
||||
):
|
||||
a, b = mul.args
|
||||
if cls._is_act_of_gate(a, module) and is_linear(b, module):
|
||||
act, gate, up = a, a.args[0], b
|
||||
elif cls._is_act_of_gate(b, module) and is_linear(a, module):
|
||||
act, gate, up = b, b.args[0], a
|
||||
else:
|
||||
continue
|
||||
if (
|
||||
all(len(args) == 1 for args in (gate.args, up.args))
|
||||
and isinstance(x := gate.args[0], fx.Node)
|
||||
and x is up.args[0]
|
||||
):
|
||||
return act, gate, up, mul
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _get_act_and_mul_name(act: nn.Module) -> str | None:
|
||||
"""Get the name of `act` if it has an `...AndMul` equivalent."""
|
||||
for name in CLS2ACT.get(type(act), []):
|
||||
if name in ACT_AND_MUL_NAMES:
|
||||
return name
|
||||
# nn.GELU is not in ACT2CLS, but could be in model code
|
||||
if type(act) is nn.GELU:
|
||||
return "gelu_pytorch_tanh" if act.approximate == "tanh" else "gelu"
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _get_act_and_mul(cls, act: nn.Module) -> nn.Module:
|
||||
"""Get the `...AndMul` equivalent of a Transformers activation module."""
|
||||
if name := cls._get_act_and_mul_name(act):
|
||||
return get_act_and_mul_fn(name)
|
||||
raise ValueError(f"No AndMul equivalent for {type(act)}")
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "GLUFuser | None":
|
||||
if (glu_nodes := cls._get_glu_nodes(graph, module)) is None:
|
||||
return None
|
||||
act_node, gate_node, up_node, mul_node = glu_nodes
|
||||
|
||||
predicate = lambda n: is_linear(n, module) and peel(n.args[0]) is mul_node
|
||||
down_node = find_node(graph, predicate)
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
act_name=act_node.target,
|
||||
linear_names=(gate_node.target, up_node.target),
|
||||
down_name=down_node.target if down_node is not None else None,
|
||||
)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace `act(gate(x)) * up(x)` with `act(gate_up(x))` in source."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
act_call = single_self_call(funcdef, self.act_name)
|
||||
calls = self._unguarded_calls(funcdef, (self.gate_name, self.up_name))
|
||||
gate_call, up_call = calls
|
||||
if act_call.args[0] is not gate_call:
|
||||
raise ValueError("activation does not directly wrap the gate")
|
||||
if ast.dump(gate_call.args[0]) != ast.dump(up_call.args[0]):
|
||||
raise ValueError("gate and up inputs are written differently")
|
||||
muls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.BinOp)
|
||||
and isinstance(node.op, ast.Mult)
|
||||
and {id(node.left), id(node.right)} == {id(act_call), id(up_call)}
|
||||
]
|
||||
if len(muls) != 1:
|
||||
raise ValueError("no multiply of the activation and up projection")
|
||||
|
||||
# act(gate(x)) * up(x) -> act(gate_up(x))
|
||||
assert isinstance(gate_call.func, ast.Attribute)
|
||||
gate_call.func.attr = self.merged_name
|
||||
replace_expr(funcdef, muls[0], act_call)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
if not super().validate(module, vllm_config):
|
||||
return False
|
||||
# An AndMul kernel splits its input in half, so gate and up must match.
|
||||
gate = module.get_submodule(self.gate_name)
|
||||
up = module.get_submodule(self.up_name)
|
||||
if gate.out_features != up.out_features:
|
||||
logger.debug("%s and %s differ in size; skipping fusion", gate, up)
|
||||
return False
|
||||
act = module.get_submodule(self.act_name)
|
||||
if self._get_act_and_mul_name(act) is None:
|
||||
logger.debug("%s has no AndMul equivalent; skipping fusion", act)
|
||||
return False
|
||||
return True
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
super().update_attrs(module, prefix, vllm_config)
|
||||
act_fn = self._get_act_and_mul(module.get_submodule(self.act_name))
|
||||
setattr(module, self.act_name, act_fn)
|
||||
# If there is a down projection, we know it must be rowwise.
|
||||
if self.down_name is not None:
|
||||
down_prefix = maybe_prefix(prefix, self.down_name)
|
||||
down = module.get_submodule(self.down_name)
|
||||
new_down = replace_linear_class(
|
||||
down, "rowwise", vllm_config.quant_config, prefix=down_prefix
|
||||
)
|
||||
setattr(module, self.down_name, new_down)
|
||||
log_replacement(down_prefix, down, new_down)
|
||||
@@ -1,145 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Fuser for parallel linear projections."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
block_chain,
|
||||
compile_forward,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MergedColumnParallelFuser(StackedFuser):
|
||||
"""Fuser for merging column-parallel linear projections."""
|
||||
|
||||
linear_names: tuple[str, ...]
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
@property
|
||||
def merged_name(self) -> str:
|
||||
"""Programmatic name for the merged projection, based on the original names."""
|
||||
# len is used to disambiguate names like `a_b` + `c` vs `a` + `b_c`
|
||||
parts = "_".join(f"{len(name)}_{name}" for name in self.linear_names)
|
||||
return f"merged_proj_{parts}"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
return [(name, index) for index, name in enumerate(self.linear_names)]
|
||||
|
||||
@staticmethod
|
||||
def _names(group: list[fx.Node]) -> tuple[str, ...] | None:
|
||||
names = tuple(str(node.target) for node in group)
|
||||
if len(set(names)) != len(names) or any("." in name for name in names):
|
||||
return None
|
||||
return names
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> "MergedColumnParallelFuser | None":
|
||||
"""Fuse the module's sibling linears when there is only one such group."""
|
||||
by_input: dict[fx.Node, list[fx.Node]] = {}
|
||||
for node in graph.nodes:
|
||||
if (
|
||||
is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
# Like QKVFuser/PackedQKVFuser: this fuser has no head or
|
||||
# TP-replication awareness, so it must not absorb a QKV-shaped
|
||||
# pattern (e.g. behind a norm) a more specific fuser declined.
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
by_input.setdefault(node.args[0], []).append(node)
|
||||
# A group whose members are not distinct direct children is dropped:
|
||||
# the source rewrite addresses each projection as `self.<name>` once.
|
||||
groups = [
|
||||
(group, names)
|
||||
for group in by_input.values()
|
||||
if len(group) >= 2 and (names := cls._names(group)) is not None
|
||||
]
|
||||
if len(groups) > 1:
|
||||
logger.debug(
|
||||
"%s has %d fusable sibling-linear groups; skipping fusion "
|
||||
"since which one to merge is ambiguous",
|
||||
type(module).__name__,
|
||||
len(groups),
|
||||
)
|
||||
if len(groups) != 1:
|
||||
return None
|
||||
_, names = groups[0]
|
||||
fuser = cls(source_cls=type(module).__name__, linear_names=names)
|
||||
return None if hasattr(module, fuser.merged_name) else fuser
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace the parallel calls with one merged call and split."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
calls = self._unguarded_calls(funcdef, self.linear_names)
|
||||
if len(set(ast.dump(call.args[0]) for call in calls)) != 1:
|
||||
raise ValueError("parallel linears read different inputs")
|
||||
chains = [block_chain(funcdef.body, call) for call in calls]
|
||||
if any(not chain for chain in chains):
|
||||
raise ValueError("parallel linear calls not found in the function body")
|
||||
blocks = [chain[-1] for chain in chains]
|
||||
if len(set(id(block) for block, _ in blocks)) != 1:
|
||||
raise ValueError("parallel linear calls are in different blocks")
|
||||
|
||||
block = blocks[0][0]
|
||||
indices = [index for _, index in blocks]
|
||||
index = min(indices)
|
||||
self._check_input_stable(funcdef, module, calls, block, indices)
|
||||
|
||||
# l1(x), l2(x), ... -> merged(x).split(merged.output_sizes / merged.tp_size, -1)
|
||||
self._splice_merged_split(funcdef, calls, block, index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Check that the parallel linears are compatible for merging."""
|
||||
linear_layers = [module.get_submodule(name) for name in self.linear_names]
|
||||
tp_size = vllm_config.parallel_config.tensor_parallel_size
|
||||
return (
|
||||
len(linear_layers) >= 2
|
||||
and len(set(linear.in_features for linear in linear_layers)) == 1
|
||||
and len(set(linear.bias is None for linear in linear_layers)) == 1
|
||||
and all(linear.out_features % tp_size == 0 for linear in linear_layers)
|
||||
)
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
"""Replace the module's parallel linears with one merged projection."""
|
||||
linear_modules = [module.get_submodule(name) for name in self.linear_names]
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=linear_modules[0].in_features,
|
||||
output_sizes=[linear.out_features for linear in linear_modules],
|
||||
bias=linear_modules[0].bias is not None,
|
||||
quant_config=vllm_config.quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
for name in self.linear_names:
|
||||
delattr(module, name)
|
||||
logger.debug(
|
||||
"%s -> %s: %s",
|
||||
", ".join(f"{n}: {m}" for n, m in zip(self.linear_names, linear_modules)),
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
@@ -1,303 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MLA fuser: adapt a Transformers MLA attention module for vLLM's MLA layer."""
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import textwrap
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import MergedColumnParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import StackedFuser
|
||||
from vllm.model_executor.models.transformers.fusers.rms_norm import RMSNormFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
downstream_linear,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
replace_expr,
|
||||
returned_linear,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
Style,
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _consumes_placeholder(node: fx.Node) -> bool:
|
||||
"""Whether `node` is a linear applied directly to a `forward` input."""
|
||||
return (
|
||||
len(node.args) == 1
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
and node.args[0].op == "placeholder"
|
||||
)
|
||||
|
||||
|
||||
def _norm_size(norm: nn.Module) -> int:
|
||||
weight = getattr(norm, "weight", None)
|
||||
return weight.numel() if weight is not None else -1
|
||||
|
||||
|
||||
def _is_rms_norm(module: nn.Module) -> bool:
|
||||
"""Whether `module` computes an RMSNorm, verified by `RMSNormFuser`'s matcher."""
|
||||
# Go via `get_fuser` because it caches
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
|
||||
return get_fuser(module, RMSNormFuser) is not None
|
||||
|
||||
|
||||
def _top_level_index(funcdef: ast.FunctionDef, node: ast.AST) -> int:
|
||||
"""Index in `funcdef.body` of the top-level statement containing `node`."""
|
||||
for index, stmt in enumerate(funcdef.body):
|
||||
if any(child is node for child in ast.walk(stmt)):
|
||||
return index
|
||||
raise ValueError("node is not in the function body")
|
||||
|
||||
|
||||
def _single_expand_call(
|
||||
funcdef: ast.FunctionDef, module: nn.Module, kv_b_proj_name: str
|
||||
) -> ast.Call:
|
||||
"""The KV expansion call `self.<method>(kv_c_normed, k_pe)`: the unique
|
||||
two-argument call to a method of `module` with the expansion's signature.
|
||||
|
||||
The expansion is `kv_c_normed, k_pe -> key, value`: two inputs, every
|
||||
`return` a pair, and it applies `kv_b_proj` -- the projection `MLAAttention`
|
||||
owns after the bypass, absorbed into the attention computation.
|
||||
"""
|
||||
|
||||
def is_expansion_method(name: str) -> bool:
|
||||
method = getattr(type(module), name, None)
|
||||
if method is None:
|
||||
return False
|
||||
code = getattr(inspect.unwrap(method), "__code__", None)
|
||||
if (
|
||||
code is None
|
||||
or code.co_argcount != 3 # self, kv_c_normed, k_pe
|
||||
or kv_b_proj_name not in code.co_names
|
||||
):
|
||||
return False
|
||||
try:
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(method)))
|
||||
except (OSError, TypeError):
|
||||
return False
|
||||
returns = [node for node in ast.walk(tree) if isinstance(node, ast.Return)]
|
||||
return bool(returns) and all(
|
||||
isinstance(ret.value, ast.Tuple) and len(ret.value.elts) == 2
|
||||
for ret in returns
|
||||
)
|
||||
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and isinstance(node.func.value, ast.Name)
|
||||
and node.func.value.id == "self"
|
||||
and len(node.args) == 2
|
||||
and not node.keywords
|
||||
and is_expansion_method(node.func.attr)
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError("expected exactly one KV expansion method call")
|
||||
return calls[0]
|
||||
|
||||
|
||||
@dataclass
|
||||
class MLAFuser(StackedFuser):
|
||||
"""Fuser for the MLA attention pattern."""
|
||||
|
||||
q_proj_name: str | None
|
||||
q_a_proj_name: str | None
|
||||
q_a_layernorm_name: str | None
|
||||
q_b_proj_name: str | None
|
||||
kv_a_proj_name: str
|
||||
kv_a_layernorm_name: str
|
||||
kv_b_proj_name: str
|
||||
o_proj_name: str | None
|
||||
merged_name: ClassVar[str] = "fused_qkv_a_proj"
|
||||
merged_cls_name: ClassVar[str] = "MergedColumnParallelLinear"
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
info_str = (
|
||||
f"Fused: {name} ({self.source_cls}) -> MLAAttention (attention interface)"
|
||||
)
|
||||
if self.q_a_proj_name is not None:
|
||||
info_str += "; " + super().info(name).removeprefix("Fused: ")
|
||||
return info_str
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
"""`q_a_proj` and `kv_a_proj_with_mqa` stack into one down-projection."""
|
||||
if self.q_a_proj_name is None:
|
||||
return []
|
||||
return [(self.q_a_proj_name, 0), (self.kv_a_proj_name, 1)]
|
||||
|
||||
@property
|
||||
def packed_modules_mapping(self) -> dict[str, list[str]]:
|
||||
if self.q_a_proj_name is None:
|
||||
return {}
|
||||
return super().packed_modules_mapping
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "MLAFuser | None":
|
||||
# Find all `rms_norm(linear(placeholder))` chains.
|
||||
chains = []
|
||||
for node in graph.nodes:
|
||||
if node.op != "call_module" or is_linear(node, module) or not node.args:
|
||||
continue
|
||||
source = upstream_linear(node.args[0], module)
|
||||
if source is None or not _consumes_placeholder(source):
|
||||
continue
|
||||
if not _is_rms_norm(module.get_submodule(node.target)):
|
||||
continue
|
||||
chains.append((source, node))
|
||||
|
||||
# Tell the chains apart by width.
|
||||
def is_kv_a(chain) -> bool:
|
||||
source, rms_norm = chain
|
||||
source_mod = module.get_submodule(source.target)
|
||||
rms_norm_mod = module.get_submodule(rms_norm.target)
|
||||
return source_mod.out_features != _norm_size(rms_norm_mod)
|
||||
|
||||
kv_a_chains = [chain for chain in chains if is_kv_a(chain)]
|
||||
q_a_chains = [chain for chain in chains if not is_kv_a(chain)]
|
||||
# Exactly one KV chain is MLA's signature. The query chain is optional.
|
||||
if len(kv_a_chains) != 1 or len(q_a_chains) > 1:
|
||||
return None
|
||||
kv_a_proj, kv_a_layernorm = kv_a_chains[0]
|
||||
|
||||
# Linear children claimed for a role so far; the rest resolve by elimination.
|
||||
claimed_linears = {kv_a_proj.target}
|
||||
q_proj_name = q_a_proj = q_a_layernorm = q_b_proj = None
|
||||
if q_a_chains:
|
||||
# Find `q_b_proj(q_a_layernorm(q_a_proj(placeholder)))`.
|
||||
q_a_proj, q_a_layernorm = q_a_chains[0]
|
||||
q_b_proj = downstream_linear(q_a_layernorm, module)
|
||||
if q_b_proj is None:
|
||||
return None
|
||||
claimed_linears |= {q_a_proj.target, q_b_proj.target}
|
||||
else:
|
||||
# Find `q_proj(placeholder)`.
|
||||
placeholder_linears = {
|
||||
node.target
|
||||
for node in graph.nodes
|
||||
if is_linear(node, module) and _consumes_placeholder(node)
|
||||
}
|
||||
if len(q_proj_candidates := placeholder_linears - claimed_linears) != 1:
|
||||
return None
|
||||
q_proj_name = next(iter(q_proj_candidates))
|
||||
claimed_linears.add(q_proj_name)
|
||||
|
||||
# Find `kv_b_proj(kv_a_layernorm(...))`.
|
||||
kv_b_proj = downstream_linear(kv_a_layernorm, module)
|
||||
if kv_b_proj is None or kv_b_proj.target in claimed_linears:
|
||||
return None
|
||||
claimed_linears.add(kv_b_proj.target)
|
||||
|
||||
# Find `o_proj` if it is returned by the forward graph.
|
||||
o_proj_name = returned_linear(graph, module)
|
||||
if o_proj_name in claimed_linears:
|
||||
o_proj_name = None
|
||||
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
q_proj_name=q_proj_name,
|
||||
q_a_proj_name=q_a_proj.target if q_a_proj else None,
|
||||
q_a_layernorm_name=q_a_layernorm.target if q_a_layernorm else None,
|
||||
q_b_proj_name=q_b_proj.target if q_b_proj else None,
|
||||
kv_a_proj_name=kv_a_proj.target,
|
||||
kv_a_layernorm_name=kv_a_layernorm.target,
|
||||
kv_b_proj_name=kv_b_proj.target,
|
||||
o_proj_name=o_proj_name,
|
||||
)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
return vllm_config.model_config.use_mla
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Merge `q_a_proj` and `kv_a_proj` into one fused down proj then split.
|
||||
Bypass the KV expansion method so the compressed latent reaches the `vllm_mla`
|
||||
attention interface unexpanded."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
if (q_a_proj_name := self.q_a_proj_name) is not None:
|
||||
# q_a_proj is usually inside the `else` of `if self.q_lora_rank is None`.
|
||||
# The fused call is inserted at the top-level statement preceding both.
|
||||
names = [q_a_proj_name, self.kv_a_proj_name]
|
||||
calls = self._unguarded_calls(funcdef, names)
|
||||
if ast.dump(calls[0].args[0]) != ast.dump(calls[1].args[0]):
|
||||
raise ValueError("down-projections read different inputs")
|
||||
indices = [_top_level_index(funcdef, call) for call in calls]
|
||||
self._check_input_stable(funcdef, module, calls, funcdef.body, indices)
|
||||
self._splice_merged_split(funcdef, calls, funcdef.body, min(indices))
|
||||
|
||||
# Transformers expands the latent into full key/value in a dedicated method.
|
||||
# `MLAAttention` consumes the latent directly (absorbing `kv_b_proj`),
|
||||
# so replace the expansion call with its own arguments so `kv_c_normed, k_pe`
|
||||
# flow to the interface in place of `key, value`.
|
||||
expand_call = _single_expand_call(funcdef, module, self.kv_b_proj_name)
|
||||
replace_expr(
|
||||
funcdef, expand_call, ast.Tuple(elts=list(expand_call.args), ctx=ast.Load())
|
||||
)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def update_attrs(self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"):
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
def replace_linear_by_name(name: str, style: Style):
|
||||
linear = module.get_submodule(name)
|
||||
replacement = replace_linear_class(
|
||||
linear, style, quant_config, prefix=maybe_prefix(prefix, name)
|
||||
)
|
||||
setattr(module, name, replacement)
|
||||
log_replacement(maybe_prefix(prefix, name), linear, replacement)
|
||||
|
||||
if (q_a_proj_name := self.q_a_proj_name) is not None:
|
||||
# `match` sets the q-LoRA names together, or none of them
|
||||
assert self.q_b_proj_name is not None
|
||||
q_a = module.get_submodule(q_a_proj_name)
|
||||
kv_a = module.get_submodule(self.kv_a_proj_name)
|
||||
merged = MergedColumnParallelLinear(
|
||||
input_size=q_a.in_features,
|
||||
output_sizes=[q_a.out_features, kv_a.out_features],
|
||||
bias=q_a.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
disable_tp=True,
|
||||
)
|
||||
logger.debug(
|
||||
"%s: %s, %s: %s -> %s: %s",
|
||||
q_a_proj_name,
|
||||
q_a,
|
||||
self.kv_a_proj_name,
|
||||
kv_a,
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
# The rewritten forward calls the merged projection instead.
|
||||
delattr(module, q_a_proj_name)
|
||||
delattr(module, self.kv_a_proj_name)
|
||||
replace_linear_by_name(self.q_b_proj_name, "colwise")
|
||||
else:
|
||||
assert self.q_proj_name is not None
|
||||
replace_linear_by_name(self.kv_a_proj_name, "replicate")
|
||||
replace_linear_by_name(self.q_proj_name, "colwise")
|
||||
|
||||
replace_linear_by_name(self.kv_b_proj_name, "colwise")
|
||||
# MLAAttention calls kv_b_proj and expects vLLM's default return_bias=True
|
||||
module.get_submodule(self.kv_b_proj_name).return_bias = True
|
||||
if self.o_proj_name is not None:
|
||||
replace_linear_by_name(self.o_proj_name, "rowwise")
|
||||
@@ -1,439 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""MoE fuser: run an HF MoE block's gate, experts and shared experts in `FusedMoE`."""
|
||||
|
||||
import ast
|
||||
import functools
|
||||
import inspect
|
||||
import textwrap
|
||||
import types
|
||||
from collections.abc import Iterable, Iterator
|
||||
from dataclasses import dataclass
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.fused_moe import GateLinear
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
is_complete,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import named_state
|
||||
from vllm.model_executor.models.utils import maybe_prefix, sequence_parallel_chunk
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _own_returns(node: ast.AST) -> Iterator[ast.Return]:
|
||||
"""`return` statements in `node`'s own scope, not in nested functions."""
|
||||
stack = list(ast.iter_child_nodes(node))
|
||||
while stack:
|
||||
child = stack.pop()
|
||||
if isinstance(child, ast.Return):
|
||||
yield child
|
||||
elif not isinstance(child, (ast.FunctionDef, ast.AsyncFunctionDef, ast.Lambda)):
|
||||
stack.extend(ast.iter_child_nodes(child))
|
||||
|
||||
|
||||
def _returns_tuple(cls: type[nn.Module]) -> bool:
|
||||
"""Does `cls.forward()` return a tuple?"""
|
||||
try:
|
||||
source = textwrap.dedent(inspect.getsource(inspect.unwrap(cls.forward)))
|
||||
forward = ast.parse(source).body[0]
|
||||
except (OSError, SyntaxError, TypeError, IndexError):
|
||||
return True
|
||||
# Names bound to a tuple literal, e.g. `out = hidden, logits` then `return out`.
|
||||
tuple_names = {
|
||||
target.id
|
||||
for node in ast.walk(forward)
|
||||
if isinstance(node, ast.Assign) and isinstance(node.value, ast.Tuple)
|
||||
for target in node.targets
|
||||
if isinstance(target, ast.Name)
|
||||
}
|
||||
|
||||
def yields_tuple(value: ast.expr | None) -> bool:
|
||||
if isinstance(value, ast.Tuple):
|
||||
return True
|
||||
if isinstance(value, ast.Name):
|
||||
return value.id in tuple_names
|
||||
if isinstance(value, ast.IfExp):
|
||||
return yields_tuple(value.body) or yields_tuple(value.orelse)
|
||||
return False
|
||||
|
||||
return any(yields_tuple(ret.value) for ret in _own_returns(forward))
|
||||
|
||||
|
||||
def _is_scalar_gate(module: nn.Module) -> bool:
|
||||
"""A linear projecting to a single logit (the shared-expert sigmoid gate)."""
|
||||
weight = getattr(module, "weight", None)
|
||||
return (
|
||||
isinstance(module, nn.Linear)
|
||||
and weight is not None
|
||||
and weight.ndim == 2
|
||||
and weight.shape[0] == 1
|
||||
)
|
||||
|
||||
|
||||
def _forced_dtype(nodes: Iterable[fx.Node]) -> torch.dtype | None:
|
||||
"""The floating dtype `nodes` cast to, if any.
|
||||
|
||||
Computations that must run in higher precision say so in their forward (e.g.
|
||||
`hidden_states.type(torch.float32)`), so the dtype is readable from the graph
|
||||
even when the config does not name it."""
|
||||
for node in nodes:
|
||||
if node.op not in ("call_method", "call_function"):
|
||||
continue
|
||||
name = str(node.target).rsplit(".", 1)[-1]
|
||||
if name == "float":
|
||||
return torch.float32
|
||||
if name in ("to", "type"):
|
||||
for arg in (*node.args[1:], *node.kwargs.values()):
|
||||
if isinstance(arg, torch.dtype) and arg.is_floating_point:
|
||||
return arg
|
||||
return None
|
||||
|
||||
|
||||
def _reaches(node: fx.Node, key: str) -> set[fx.Node]:
|
||||
"""Returns the set of nodes reachable from `node` by following `key` edges."""
|
||||
seen: set[fx.Node] = set()
|
||||
stack = [node]
|
||||
while stack:
|
||||
n = stack.pop()
|
||||
if n in seen:
|
||||
continue
|
||||
seen.add(n)
|
||||
stack.extend(getattr(n, key))
|
||||
return seen
|
||||
|
||||
|
||||
def _renormalizes(topk: fx.Node) -> bool:
|
||||
"""Are the top-k weights divided by their own sum?
|
||||
|
||||
Every HF router spells renormalization as a division by a sum over the
|
||||
selection, optionally offset by an epsilon."""
|
||||
downstream = _reaches(topk, "users")
|
||||
for node in downstream:
|
||||
if not (is_op(node, "div") or is_op(node, "truediv")) or len(node.args) < 2:
|
||||
continue
|
||||
denominator = node.args[1]
|
||||
if not isinstance(denominator, fx.Node):
|
||||
continue
|
||||
cone = _reaches(denominator, "all_input_nodes")
|
||||
if any(is_op(n, "sum") and n in downstream for n in cone):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _unwrap(node: object, moe_block: nn.Module) -> object:
|
||||
"""Strip reshapes, dtype casts and `nn.Identity` calls."""
|
||||
while True:
|
||||
node = peel(node)
|
||||
if (
|
||||
is_op(node, "reshape")
|
||||
or is_op(node, "view")
|
||||
or (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and isinstance(moe_block.get_submodule(node.target), nn.Identity)
|
||||
)
|
||||
):
|
||||
node = node.args[0]
|
||||
else:
|
||||
return node
|
||||
|
||||
|
||||
def _reads_input(node: object, x: fx.Node, moe_block: nn.Module) -> TypeIs[fx.Node]:
|
||||
"""Is `node` a child module of `moe_block` called on just `x`?"""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_module"
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and _unwrap(node.args[0], moe_block) is x
|
||||
)
|
||||
|
||||
|
||||
class SharedExpertMLP(nn.Module):
|
||||
"""Wraps an HF shared expert, applying the output gating it is paired with."""
|
||||
|
||||
def __init__(self, shared_experts: nn.Module, gate: nn.Module | None = None):
|
||||
super().__init__()
|
||||
self.shared_experts = shared_experts
|
||||
self.gate = gate
|
||||
|
||||
def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
out = self.shared_experts(hidden_states)
|
||||
if self.gate is not None:
|
||||
out = torch.sigmoid(self.gate(hidden_states)[0]) * out
|
||||
return out
|
||||
|
||||
|
||||
def _moe_block_forward(self: nn.Module, hidden_states: torch.Tensor) -> torch.Tensor:
|
||||
"""Standard MoE block forward.
|
||||
|
||||
Routing and any shared experts are handled inside `self.experts: MoERunner`."""
|
||||
orig_shape = hidden_states.shape
|
||||
hidden_states = hidden_states.reshape(-1, orig_shape[-1])
|
||||
num_tokens = hidden_states.shape[0]
|
||||
is_sequence_parallel = self.experts.moe_config.is_sequence_parallel
|
||||
if is_sequence_parallel:
|
||||
hidden_states = sequence_parallel_chunk(hidden_states)
|
||||
out = self.experts(hidden_states, router_logits=hidden_states)
|
||||
if is_sequence_parallel:
|
||||
out = tensor_model_parallel_all_gather(out, 0)[:num_tokens]
|
||||
return out.reshape(orig_shape)
|
||||
|
||||
|
||||
@dataclass
|
||||
class MoEBlockFuser:
|
||||
"""Fuser for MoE block `experts`, `gate` and `shared_experts` (optional)."""
|
||||
|
||||
gate_name: str
|
||||
scoring_func: str | None
|
||||
"""`None` if routing stays in the Transformers gate."""
|
||||
shared_name: str | None
|
||||
shared_gate_name: str | None
|
||||
router_dtype: torch.dtype | None = None
|
||||
renormalize: bool | None = None
|
||||
|
||||
@staticmethod
|
||||
def _match_router(
|
||||
gate: nn.Module,
|
||||
) -> tuple[str, torch.dtype | None, bool | None] | None:
|
||||
"""Matches `topk(score(linear(x)))` or `score(topk(linear(x)))`, `score` being
|
||||
`softmax`/`sigmoid`.
|
||||
|
||||
Returns the scoring function, the dtype the router computes in, and
|
||||
whether the top-k weights are renormalized (`None` if the trace was cut
|
||||
short before it could say)."""
|
||||
state = {name for name, _ in named_state(gate)}
|
||||
if "weight" not in state or state - {"weight", "e_score_correction_bias"}:
|
||||
return None
|
||||
graph = trace(gate)
|
||||
if graph is None:
|
||||
return None
|
||||
# The routing top-k is the last one; any earlier one scores expert groups.
|
||||
topks = [node for node in graph.nodes if is_op(node, "topk")]
|
||||
if not topks:
|
||||
return None
|
||||
topk = topks[-1]
|
||||
is_scorer = lambda n: is_op(n, "softmax") or is_op(n, "sigmoid")
|
||||
post_topk = False
|
||||
# Find all scoring functions that reach the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "all_input_nodes") if is_scorer(n)]
|
||||
if not scorers:
|
||||
# Find all scoring functions that are users of the top-k node.
|
||||
scorers = [n for n in _reaches(topk, "users") if is_scorer(n)]
|
||||
post_topk = True
|
||||
if len(scorers) != 1:
|
||||
return None
|
||||
scorer = scorers[0]
|
||||
logits_cone = _reaches(scorer, "all_input_nodes")
|
||||
if not any(is_op(n, "linear") for n in logits_cone):
|
||||
return None
|
||||
is_softmax = is_op(scorer, "softmax")
|
||||
# vLLM always scores every expert, so matching a router that softmaxes
|
||||
# the selection alone takes renormalizing. Otherwise look for the
|
||||
# division, reading its absence as "no" only from a complete trace.
|
||||
renormalize: bool | None
|
||||
if (post_topk and is_softmax) or _renormalizes(topk):
|
||||
renormalize = True
|
||||
else:
|
||||
renormalize = False if is_complete(graph) else None
|
||||
scoring_func = "softmax" if is_softmax else "sigmoid"
|
||||
return scoring_func, _forced_dtype(logits_cone), renormalize
|
||||
|
||||
@staticmethod
|
||||
def _match_experts(
|
||||
graph: fx.Graph, experts_name: str, moe_block: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Matches `experts(x, ids, weights)` with `_, weights, ids = gate(x)`, `x`
|
||||
being the block's input.
|
||||
|
||||
Returns the experts call, `x` and the gate call."""
|
||||
experts = find_node(
|
||||
graph, lambda n: n.op == "call_module" and n.target == experts_name
|
||||
)
|
||||
if experts is None:
|
||||
return None
|
||||
# Bind by signature so keyword calls match too. Fail closed on any other
|
||||
# argument: the rewritten forward would drop it.
|
||||
forward = inspect.signature(moe_block.get_submodule(experts_name).forward)
|
||||
try:
|
||||
bound = forward.bind(*experts.args, **experts.kwargs)
|
||||
except TypeError:
|
||||
return None
|
||||
if len(bound.arguments) != 3:
|
||||
return None
|
||||
x, ids, weights = (_unwrap(a, moe_block) for a in bound.arguments.values())
|
||||
if not (isinstance(x, fx.Node) and x.op == "placeholder"):
|
||||
return None
|
||||
if not (is_op(ids, "getitem") and is_op(weights, "getitem")):
|
||||
return None
|
||||
gate = ids.args[0]
|
||||
if not (
|
||||
_reads_input(gate, x, moe_block)
|
||||
and ids.args == (gate, 2)
|
||||
and weights.args == (gate, 1)
|
||||
):
|
||||
return None
|
||||
return experts, x, gate
|
||||
|
||||
@staticmethod
|
||||
def _match_shared_experts(
|
||||
node: object, x: fx.Node, moe_block: nn.Module
|
||||
) -> tuple[str, str | None] | None:
|
||||
"""Matches `shared(x)` or `sigmoid(shared_gate(x)) * shared(x)`.
|
||||
|
||||
Returns the shared expert's name and its optional gate's name."""
|
||||
if _reads_input(node, x, moe_block):
|
||||
return node.target, None
|
||||
if not (is_op(node, "mul") and len(node.args) == 2):
|
||||
return None
|
||||
for gated, shared in (node.args, node.args[::-1]):
|
||||
gated, shared = peel(gated), peel(shared)
|
||||
if not (is_op(gated, "sigmoid") and _reads_input(shared, x, moe_block)):
|
||||
continue
|
||||
gate = peel(gated.args[0])
|
||||
if _reads_input(gate, x, moe_block) and _is_scalar_gate(
|
||||
moe_block.get_submodule(gate.target)
|
||||
):
|
||||
return shared.target, gate.target
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def match(
|
||||
cls,
|
||||
moe_block: nn.Module,
|
||||
experts_name: str,
|
||||
divides_shared_output: bool = False,
|
||||
) -> "MoEBlockFuser | None":
|
||||
"""Matches a block returning `experts(x, ids, weights) [* c] [+ shared(x)]`
|
||||
with `_, weights, ids = gate(x)`, `x` being the block's input, up to
|
||||
reshapes, casts and `nn.Identity`.
|
||||
|
||||
Args:
|
||||
moe_block: The MoE block.
|
||||
experts_name: The name of the experts in `moe_block`.
|
||||
divides_shared_output: Whether vLLM routing would divide the shared
|
||||
expert's output by `routed_scaling_factor` (fp16 overflow
|
||||
protection), which Transformers decoder layers do not undo.
|
||||
|
||||
Returns:
|
||||
A fuser if more than the experts can run in vLLM, else `None`.
|
||||
|
||||
"""
|
||||
# Standard MoE block returns a single tensor.
|
||||
if _returns_tuple(type(moe_block)):
|
||||
return None
|
||||
if (graph := trace(moe_block)) is None:
|
||||
return None
|
||||
if (matched := cls._match_experts(graph, experts_name, moe_block)) is None:
|
||||
return None
|
||||
experts, x, gate = matched
|
||||
unwrap = functools.partial(_unwrap, moe_block=moe_block)
|
||||
gate_name = gate.target
|
||||
router = cls._match_router(getattr(moe_block, gate_name))
|
||||
scoring_func, router_dtype, renormalize = router or (None, None, None)
|
||||
|
||||
def is_routed(n: object) -> bool:
|
||||
return n is experts or (
|
||||
is_op(n, "mul")
|
||||
and not isinstance(n.args[1], fx.Node)
|
||||
and unwrap(n.args[0]) is experts
|
||||
)
|
||||
|
||||
# Output: the experts' output, plus the shared expert's if any.
|
||||
routed = out = unwrap(output_value(graph))
|
||||
shared = None
|
||||
if not is_routed(out):
|
||||
if not (is_op(out, "add") and len(out.args) == 2):
|
||||
return None
|
||||
routed, shared = map(unwrap, out.args)
|
||||
if is_routed(shared):
|
||||
routed, shared = shared, routed
|
||||
if not is_routed(routed):
|
||||
return None
|
||||
shared_name = shared_gate_name = None
|
||||
if shared is not None:
|
||||
if (match := cls._match_shared_experts(shared, x, moe_block)) is None:
|
||||
return None
|
||||
shared_name, shared_gate_name = match
|
||||
# Fail closed: `rewrite_forward` runs only the experts and the detected
|
||||
# shared expert, so any other stateful child would be dropped.
|
||||
accounted = {experts_name, gate_name, shared_name, shared_gate_name}
|
||||
for name, child in moe_block.named_children():
|
||||
if name not in accounted and next(named_state(child), None) is not None:
|
||||
return None
|
||||
if scoring_func is not None and shared_name and divides_shared_output:
|
||||
logger.warning_once(
|
||||
"%s could be fused but routing it in vLLM would apply "
|
||||
"`routed_scaling_factor` by dividing the shared expert output, "
|
||||
"which only fp16 overflow protection expects the decoder layer to "
|
||||
"compensate for. Falling back to routing in Transformers; run in "
|
||||
"bfloat16 to fuse it.",
|
||||
type(moe_block).__name__,
|
||||
)
|
||||
scoring_func = router_dtype = renormalize = None
|
||||
# Without vLLM routing, only a shared expert can move to vLLM, and only
|
||||
# if the block does not scale the experts' output (vLLM routing does).
|
||||
if scoring_func is None and (shared_name is None or routed is not experts):
|
||||
return None
|
||||
return cls(
|
||||
gate_name,
|
||||
scoring_func,
|
||||
shared_name,
|
||||
shared_gate_name,
|
||||
router_dtype,
|
||||
renormalize,
|
||||
)
|
||||
|
||||
def gate(
|
||||
self, moe_block: nn.Module, prefix: str, out_dtype: torch.dtype | None = None
|
||||
) -> GateLinear:
|
||||
"""Rebuild the HF gate as a `GateLinear` for vLLM's fused MoE."""
|
||||
hf_gate = getattr(moe_block, self.gate_name)
|
||||
num_experts, hidden_size = hf_gate.weight.shape
|
||||
gate = GateLinear(
|
||||
hidden_size,
|
||||
num_experts,
|
||||
bias=False,
|
||||
out_dtype=out_dtype or self.router_dtype,
|
||||
prefix=maybe_prefix(prefix, self.gate_name),
|
||||
)
|
||||
if (bias := getattr(hf_gate, "e_score_correction_bias", None)) is not None:
|
||||
gate.register_buffer("e_score_correction_bias", bias)
|
||||
setattr(moe_block, self.gate_name, gate)
|
||||
return gate
|
||||
|
||||
def shared_experts(
|
||||
self, moe_block: nn.Module, prefix: str
|
||||
) -> SharedExpertMLP | None:
|
||||
"""Build the HF shared expert (and its optional gate)
|
||||
as a `SharedExpertMLP` for vLLM's fused MoE."""
|
||||
if self.shared_name is None:
|
||||
return None
|
||||
shared_experts = getattr(moe_block, self.shared_name)
|
||||
gate = None
|
||||
if self.shared_gate_name is not None:
|
||||
hf_gate = getattr(moe_block, self.shared_gate_name)
|
||||
gate = ReplicatedLinear(
|
||||
hf_gate.in_features,
|
||||
hf_gate.out_features,
|
||||
bias=hf_gate.bias is not None,
|
||||
prefix=maybe_prefix(prefix, self.shared_gate_name),
|
||||
)
|
||||
setattr(moe_block, self.shared_gate_name, gate)
|
||||
return SharedExpertMLP(shared_experts, gate)
|
||||
|
||||
def rewrite_forward(self, moe_block: nn.Module) -> None:
|
||||
"""Rewrite `moe_block.forward` to route through vLLM's fused MoE."""
|
||||
moe_block.forward = types.MethodType(_moe_block_forward, moe_block)
|
||||
@@ -1,164 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Packed-QKV fuser: `c_attn(x).split((q, kv, kv))` -> a `QKVParallelLinear`."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
RewriteFuser,
|
||||
fused_head_size,
|
||||
local_output_sizes,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
compile_forward,
|
||||
is_method,
|
||||
recover_forward,
|
||||
returned_linear,
|
||||
upstream_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PackedQKVFuser(RewriteFuser):
|
||||
"""Fuser for attention with q, k and v packed into one projection."""
|
||||
|
||||
qkv_name: str
|
||||
o_name: str | None
|
||||
q_size: int
|
||||
kv_size: int
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
return (
|
||||
f"Fused: {self.qkv_name} ({name}: {self.source_cls}) -> QKVParallelLinear"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _packed_sizes(node: fx.Node) -> tuple[int, int] | None:
|
||||
"""`(q, kv)` from a `split((q, kv, kv), ...)` call, if it is one."""
|
||||
if not is_method(node, "split") or len(node.args) < 2:
|
||||
return None
|
||||
sizes = node.args[1]
|
||||
if not isinstance(sizes, (tuple, list)) or len(sizes) != 3:
|
||||
return None
|
||||
if not all(isinstance(size, int) for size in sizes):
|
||||
return None
|
||||
q_size, k_size, v_size = sizes
|
||||
if k_size != v_size or q_size < k_size:
|
||||
return None
|
||||
return q_size, k_size
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "PackedQKVFuser | None":
|
||||
for node in graph.nodes:
|
||||
if (sizes := cls._packed_sizes(node)) is None:
|
||||
continue
|
||||
q_size, kv_size = sizes
|
||||
qkv_node = upstream_linear(node.args[0], module)
|
||||
if qkv_node is None:
|
||||
continue
|
||||
qkv_name = str(qkv_node.target)
|
||||
# The split must consume the whole projection.
|
||||
if module.get_submodule(qkv_name).out_features != q_size + 2 * kv_size:
|
||||
continue
|
||||
# o_proj produces the module's output and consumes the query width.
|
||||
o_name = returned_linear(graph, module)
|
||||
if o_name == qkv_name or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features != q_size
|
||||
):
|
||||
o_name = None
|
||||
return cls(
|
||||
source_cls=type(module).__name__,
|
||||
qkv_name=qkv_name,
|
||||
o_name=o_name,
|
||||
q_size=q_size,
|
||||
kv_size=kv_size,
|
||||
)
|
||||
return None
|
||||
|
||||
def _split_call(self, funcdef: ast.FunctionDef) -> ast.Call:
|
||||
"""The unique `self.<qkv_name>(...)....split((a, b, c), ...)` call."""
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr == "split"
|
||||
and node.args
|
||||
and isinstance(node.args[0], (ast.Tuple, ast.List))
|
||||
and len(node.args[0].elts) == 3
|
||||
and any(
|
||||
isinstance(inner, ast.Attribute) and inner.attr == self.qkv_name
|
||||
for inner in ast.walk(node.func.value)
|
||||
)
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError(f"{self.qkv_name} has {len(calls)} three-way splits")
|
||||
return calls[0]
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Rewrite the split sizes to the sharded projection's per-rank widths."""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
split = self._split_call(funcdef)
|
||||
# (q, kv, kv) -> [s // qkv.tp_size for s in qkv.output_sizes]
|
||||
sections = local_output_sizes(self.qkv_name)
|
||||
split.args[0] = ast.parse(sections, mode="eval").body
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible with a head-sharded packed GEMM."""
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
compatible = (
|
||||
self.q_size % head_size == 0
|
||||
and self.kv_size % head_size == 0
|
||||
and qkv.out_features == self.q_size + 2 * self.kv_size
|
||||
)
|
||||
if not compatible:
|
||||
logger.debug("%s is not compatible with packed QKV fusion", type(module))
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
qkv_prefix = maybe_prefix(prefix, self.qkv_name)
|
||||
qkv = module.get_submodule(self.qkv_name)
|
||||
merged = QKVParallelLinear(
|
||||
hidden_size=qkv.in_features,
|
||||
head_size=head_size,
|
||||
total_num_heads=self.q_size // head_size,
|
||||
total_num_kv_heads=self.kv_size // head_size,
|
||||
bias=qkv.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=qkv_prefix,
|
||||
return_bias=False,
|
||||
)
|
||||
setattr(module, self.qkv_name, merged)
|
||||
log_replacement(qkv_prefix, qkv, merged)
|
||||
# If there is an output projection, we know it must be rowwise.
|
||||
if self.o_name is not None:
|
||||
o_prefix = maybe_prefix(prefix, self.o_name)
|
||||
o_proj = module.get_submodule(self.o_name)
|
||||
new_o = replace_linear_class(
|
||||
o_proj, "rowwise", quant_config, prefix=o_prefix
|
||||
)
|
||||
setattr(module, self.o_name, new_o)
|
||||
log_replacement(o_prefix, o_proj, new_o)
|
||||
@@ -1,209 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""QKV projection fuser: `q(x), k(x), v(x)` -> a fused qkv linear + split."""
|
||||
|
||||
import ast
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING, ClassVar
|
||||
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear
|
||||
from vllm.model_executor.models.transformers.fusers.base import (
|
||||
StackedFuser,
|
||||
fused_head_size,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
block_chain,
|
||||
compile_forward,
|
||||
is_linear,
|
||||
recover_forward,
|
||||
returned_linear,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.utils import (
|
||||
log_replacement,
|
||||
replace_linear_class,
|
||||
)
|
||||
from vllm.model_executor.models.utils import ShardId, maybe_prefix
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class QKVFuser(StackedFuser):
|
||||
"""Fuser for the attention QKV pattern `q(x), k(x), v(x)`."""
|
||||
|
||||
q_name: str
|
||||
k_name: str
|
||||
v_name: str
|
||||
o_name: str | None
|
||||
merged_name: ClassVar[str] = "qkv_proj"
|
||||
merged_cls_name: ClassVar[str] = "QKVParallelLinear"
|
||||
|
||||
@property
|
||||
def shards(self) -> list[tuple[str, ShardId]]:
|
||||
return [(self.q_name, "q"), (self.k_name, "k"), (self.v_name, "v")]
|
||||
|
||||
@classmethod
|
||||
def _get_qkv_nodes(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[fx.Node, fx.Node, fx.Node] | None:
|
||||
"""Search `graph` for the QKV pattern `q(x), k(x), v(x)`."""
|
||||
by_input: dict[fx.Node, list[fx.Node]] = {}
|
||||
for node in graph.nodes:
|
||||
if (
|
||||
is_linear(node, module)
|
||||
and len(node.args) == 1
|
||||
and not node.kwargs
|
||||
and isinstance(node.args[0], fx.Node)
|
||||
and node.args[0].op == "placeholder"
|
||||
):
|
||||
by_input.setdefault(node.args[0], []).append(node)
|
||||
triples = [nodes for nodes in by_input.values() if len(nodes) == 3]
|
||||
if len(triples) != 1:
|
||||
return None
|
||||
|
||||
q_node, k_node, v_node = nodes = triples[0]
|
||||
outs = [module.get_submodule(node.target).out_features for node in nodes]
|
||||
if len(set(outs)) == 2:
|
||||
# q is identified as the larger projection (GQA)
|
||||
(q_node,) = (n for n, out in zip(nodes, outs) if outs.count(out) == 1)
|
||||
k_node, v_node = (n for n, out in zip(nodes, outs) if outs.count(out) == 2)
|
||||
if module.get_submodule(q_node.target).out_features != max(outs):
|
||||
return None
|
||||
elif len(set(outs)) != 1:
|
||||
return None
|
||||
return q_node, k_node, v_node
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "QKVFuser | None":
|
||||
if (qkv_nodes := cls._get_qkv_nodes(graph, module)) is None:
|
||||
return None
|
||||
q, k, v = qkv_nodes
|
||||
names = dict(q_name=q.target, k_name=k.target, v_name=v.target)
|
||||
# o_proj produces the module's output.
|
||||
o_name = returned_linear(graph, module)
|
||||
# o_proj must be compatible with the q/k/v projections.
|
||||
if o_name in names.values() or (
|
||||
o_name is not None
|
||||
and module.get_submodule(o_name).in_features
|
||||
!= module.get_submodule(q.target).out_features
|
||||
):
|
||||
o_name = None
|
||||
names["o_name"] = o_name
|
||||
return cls(source_cls=type(module).__name__, **names)
|
||||
|
||||
def update_forward(self, module: nn.Module) -> None:
|
||||
"""Replace `q(x), k(x), v(x)` with `qkv(x).split(sizes, -1)` in source.
|
||||
|
||||
A projection may be guarded by an existence check -- a
|
||||
`self.<proj> is (not) None` comparison (e.g. Gemma 4's `v_proj`) or a bare
|
||||
truthiness test -- which is folded to its constant value (see
|
||||
`bypass_existence_guard`). The calls may sit in different branches, so the
|
||||
fused GEMM is inserted before the earliest of them, in the innermost block
|
||||
that dominates all three.
|
||||
"""
|
||||
funcdef, fn = recover_forward(type(module))
|
||||
proj_names = (self.q_name, self.k_name, self.v_name)
|
||||
calls = self._unguarded_calls(funcdef, proj_names)
|
||||
arg_dumps = {ast.dump(call.args[0]) for call in calls}
|
||||
if len(arg_dumps) != 1:
|
||||
raise ValueError("projection inputs are written differently")
|
||||
# The trace may be partial, so prove projection exclusivity in source:
|
||||
# no other linear child may consume the same input (else the matched
|
||||
# three may not be q, k and v)
|
||||
other_linears = {
|
||||
name
|
||||
for name, child in module.named_children()
|
||||
if isinstance(child, nn.Linear)
|
||||
} - set(proj_names)
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr in other_linears
|
||||
and any(ast.dump(arg) in arg_dumps for arg in node.args)
|
||||
):
|
||||
raise ValueError("another linear consumes the same input")
|
||||
|
||||
# Insert the fused GEMM before the earliest call, in the innermost block
|
||||
# common to all three (the calls may be split across branches).
|
||||
chains = [block_chain(funcdef.body, call) for call in calls]
|
||||
if any(not chain for chain in chains):
|
||||
raise ValueError("projection calls not found in the function body")
|
||||
depth = 0
|
||||
for level in zip(*chains):
|
||||
if len({id(block) for block, _ in level}) != 1:
|
||||
break
|
||||
depth += 1
|
||||
block = chains[0][depth - 1][0]
|
||||
indices = [chain[depth - 1][1] for chain in chains]
|
||||
insert_index = min(indices)
|
||||
self._check_input_stable(funcdef, module, calls, block, indices)
|
||||
|
||||
# q(x), k(x), v(x) -> q, k, v = qkv(x).split(qkv.output_sizes / qkv.tp_size, -1)
|
||||
self._splice_merged_split(funcdef, calls, block, insert_index)
|
||||
self.fused_forward = compile_forward(funcdef, fn)
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
"""Shapes must be compatible for a single merged, head-sharded GEMM."""
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
v = module.get_submodule(self.v_name)
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
compatible = (
|
||||
q.in_features == k.in_features == v.in_features
|
||||
and len({proj.bias is None for proj in (q, k, v)}) == 1
|
||||
and k.out_features == v.out_features
|
||||
and q.out_features % head_size == 0
|
||||
and k.out_features % head_size == 0
|
||||
)
|
||||
if not compatible:
|
||||
logger.debug("%s is not compatible with QKV fusion", type(module))
|
||||
return compatible
|
||||
|
||||
def update_attrs(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> None:
|
||||
quant_config = vllm_config.quant_config
|
||||
head_size = fused_head_size(module, vllm_config)
|
||||
q = module.get_submodule(self.q_name)
|
||||
k = module.get_submodule(self.k_name)
|
||||
merged = QKVParallelLinear(
|
||||
hidden_size=q.in_features,
|
||||
head_size=head_size,
|
||||
total_num_heads=q.out_features // head_size,
|
||||
total_num_kv_heads=k.out_features // head_size,
|
||||
bias=q.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, self.merged_name),
|
||||
return_bias=False,
|
||||
)
|
||||
logger.debug(
|
||||
"%s: %s, %s: %s, %s: %s -> %s: %s",
|
||||
self.q_name,
|
||||
q,
|
||||
self.k_name,
|
||||
k,
|
||||
self.v_name,
|
||||
module.get_submodule(self.v_name),
|
||||
self.merged_name,
|
||||
merged,
|
||||
)
|
||||
setattr(module, self.merged_name, merged)
|
||||
# Drop the consumed submodules so their (meta) params are not expected.
|
||||
for name in (self.q_name, self.k_name, self.v_name):
|
||||
delattr(module, name)
|
||||
# If there is an output projection, we know it must be rowwise.
|
||||
if self.o_name is not None:
|
||||
o_proj_prefix = maybe_prefix(prefix, self.o_name)
|
||||
o_proj = module.get_submodule(self.o_name)
|
||||
new_o = replace_linear_class(
|
||||
o_proj, "rowwise", quant_config, prefix=o_proj_prefix
|
||||
)
|
||||
setattr(module, self.o_name, new_o)
|
||||
log_replacement(o_proj_prefix, o_proj, new_o)
|
||||
@@ -1,297 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""RMSNorm fuser: detect the norm structurally and swap in vLLM's fused RMSNorm."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
tensor_model_parallel_all_gather,
|
||||
)
|
||||
from vllm.distributed.parallel_state import model_parallel_is_initialized
|
||||
from vllm.distributed.utils import split_tensor_along_last_dim
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models.transformers.fusers.base import BaseFuser
|
||||
from vllm.model_executor.models.transformers.fx_utils import (
|
||||
find_node,
|
||||
forward_input_count,
|
||||
is_op,
|
||||
output_value,
|
||||
peel,
|
||||
trace,
|
||||
)
|
||||
from vllm.model_executor.models.transformers.layers import (
|
||||
GemmaRMSNorm,
|
||||
RMSNorm,
|
||||
)
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _operand(node: fx.Node, index: int, name: str) -> object | None:
|
||||
"""Operand `index` of `node`, whether it was passed positionally or as `name`."""
|
||||
if len(node.args) > index:
|
||||
return node.args[index]
|
||||
return node.kwargs.get(name)
|
||||
|
||||
|
||||
def _is_squared(node: object, x: fx.Node) -> bool:
|
||||
"""`x**2`, `x.square()` or `x * x`, through any dtype casts."""
|
||||
node = peel(node)
|
||||
if is_op(node, "pow"):
|
||||
return (
|
||||
peel(_operand(node, 0, "input")) is x and _operand(node, 1, "exponent") == 2
|
||||
)
|
||||
if is_op(node, "square"):
|
||||
return peel(_operand(node, 0, "input")) is x
|
||||
if is_op(node, "mul"):
|
||||
return (
|
||||
peel(_operand(node, 0, "input")) is x
|
||||
and peel(_operand(node, 1, "other")) is x
|
||||
)
|
||||
return False
|
||||
|
||||
|
||||
def _is_inverse_sqrt(node: object) -> bool:
|
||||
"""Detect `rsqrt(v)`, or the `pow(v, -0.5)` / `v ** -0.5` spelling of it."""
|
||||
if is_op(node, "rsqrt"):
|
||||
return True
|
||||
if is_op(node, "pow"):
|
||||
return _operand(node, 1, "exponent") == -0.5
|
||||
return False
|
||||
|
||||
|
||||
def _variance_eps(rsqrt: fx.Node, x: fx.Node) -> float | None:
|
||||
"""`eps` from `rsqrt(mean(x**2, -1) + eps)`, or `None` if not that shape."""
|
||||
add = peel(_operand(rsqrt, 0, "input"))
|
||||
if not is_op(add, "add"):
|
||||
return None
|
||||
operands = [_operand(add, 0, "input"), _operand(add, 1, "other")]
|
||||
consts = [a for a in operands if isinstance(a, (int, float))]
|
||||
nodes = [a for a in operands if isinstance(a, fx.Node)]
|
||||
if len(consts) != 1 or len(nodes) != 1:
|
||||
return None
|
||||
mean = peel(nodes[0])
|
||||
if not is_op(mean, "mean"):
|
||||
return None
|
||||
if not _is_squared(_operand(mean, 0, "input"), x):
|
||||
return None
|
||||
return float(consts[0])
|
||||
|
||||
|
||||
def _is_one_plus(node: object) -> bool:
|
||||
"""`1 + weight` in either operand order (marks a zero-centered weight)."""
|
||||
node = peel(node)
|
||||
if not is_op(node, "add"):
|
||||
return False
|
||||
return any(
|
||||
isinstance(a, (int, float)) and a == 1
|
||||
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
|
||||
)
|
||||
|
||||
|
||||
def _has_trailing_compute(graph: fx.Graph, node: fx.Node) -> bool:
|
||||
"""Does the forward compute anything after `node` before returning?"""
|
||||
value = output_value(graph)
|
||||
return value is not None and peel(value) is not node
|
||||
|
||||
|
||||
class TPAwareNormMixin(nn.Module):
|
||||
"""Mixin for RMSNorms that reconstructs a TP-sharded input before normalizing."""
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
if model_parallel_is_initialized():
|
||||
self.tp_size = get_tensor_model_parallel_world_size()
|
||||
self.tp_rank = get_tensor_model_parallel_rank()
|
||||
else:
|
||||
self.tp_size, self.tp_rank = 1, 0
|
||||
|
||||
def forward(
|
||||
self, x: torch.Tensor, residual: torch.Tensor | None = None
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor]:
|
||||
if self.tp_size > 1 and x.shape[-1] < (full := self.weight.shape[0]):
|
||||
if x.shape[-1] * self.tp_size != full:
|
||||
raise ValueError(
|
||||
f"Cannot gather norm of width {full}: a TP-sharded input of "
|
||||
f"width {x.shape[-1]} does not tile it evenly across "
|
||||
f"{self.tp_size} ranks (replicated or uneven sharding)."
|
||||
)
|
||||
x = tensor_model_parallel_all_gather(x.contiguous())
|
||||
x = super().forward(x)
|
||||
splits = split_tensor_along_last_dim(x, num_partitions=self.tp_size)
|
||||
return splits[self.tp_rank]
|
||||
return super().forward(x, residual)
|
||||
|
||||
|
||||
class TPAwareRMSNorm(TPAwareNormMixin, RMSNorm):
|
||||
"""`RMSNorm` that reconstructs a TP-sharded input before normalizing."""
|
||||
|
||||
|
||||
class TPAwareGemmaRMSNorm(TPAwareNormMixin, GemmaRMSNorm):
|
||||
"""`GemmaRMSNorm` that reconstructs a TP-sharded input before normalizing."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class RMSNormFuser(BaseFuser):
|
||||
"""Fuser for RMSNorm patterns, including Gemma-style zero-centered weights."""
|
||||
|
||||
zero_centered: bool
|
||||
"""Gemma-style `(1 + weight)` scaling (weight initialised at zero)."""
|
||||
source_cls: str
|
||||
"""Class name of the norm this was matched from (for logging)."""
|
||||
eps_attr: str | None = None
|
||||
"""Attribute holding `eps`, read per instance in `fuse`."""
|
||||
eps: float | None = None
|
||||
"""`eps` itself, when it is not held in an attribute (see `_eps_source`)."""
|
||||
|
||||
def info(self, name: str) -> str:
|
||||
norm = "GemmaRMSNorm" if self.zero_centered else "RMSNorm"
|
||||
return f"Fused: {name} ({self.source_cls}) -> {norm} (CustomOp)"
|
||||
|
||||
@classmethod
|
||||
def match(cls, graph: fx.Graph, module: nn.Module) -> "RMSNormFuser | None":
|
||||
"""Match a graph to the RMSNorm pattern, returning a fuser if found."""
|
||||
if forward_input_count(type(module)) != 1:
|
||||
return None
|
||||
x = find_node(graph, lambda n: n.op == "placeholder")
|
||||
if x is None:
|
||||
return None
|
||||
# Handle native torch `rms_norm` op.
|
||||
rms_norm = find_node(graph, lambda n: is_op(n, "rms_norm"))
|
||||
if rms_norm is not None and peel(_operand(rms_norm, 0, "input")) is x:
|
||||
if _has_trailing_compute(graph, rms_norm):
|
||||
return None
|
||||
eps_attr, eps = cls._eps_source(graph, module)
|
||||
return cls(
|
||||
zero_centered=False,
|
||||
source_cls=type(module).__name__,
|
||||
eps_attr=eps_attr,
|
||||
eps=eps,
|
||||
)
|
||||
# Handle explicit `x * rsqrt(mean(x**2, -1) + eps)` pattern.
|
||||
# The rsqrt over the mean-square variance is the spine of the norm.
|
||||
rsqrt = None
|
||||
for node in graph.nodes:
|
||||
if _is_inverse_sqrt(node) and _variance_eps(node, x) is not None:
|
||||
rsqrt = node
|
||||
break
|
||||
if rsqrt is None:
|
||||
return None
|
||||
# The `x * rsqrt(...)` normalize multiply.
|
||||
normalize = find_node(
|
||||
graph,
|
||||
lambda n: is_op(n, "mul")
|
||||
and rsqrt in map(peel, (_operand(n, 0, "input"), _operand(n, 1, "other"))),
|
||||
)
|
||||
if normalize is None:
|
||||
return None
|
||||
# An optional later `weight * normalized` (or `(1 + weight) * normalized`).
|
||||
tail, zero_centered = normalize, False
|
||||
for node in graph.nodes:
|
||||
if not is_op(node, "mul") or node is normalize:
|
||||
continue
|
||||
operands = [
|
||||
peel(a)
|
||||
for a in (_operand(node, 0, "input"), _operand(node, 1, "other"))
|
||||
if isinstance(a, fx.Node)
|
||||
]
|
||||
if len(operands) == 2 and normalize in operands:
|
||||
weight = next(o for o in operands if o is not normalize)
|
||||
tail, zero_centered = node, _is_one_plus(weight)
|
||||
break
|
||||
# The norm must be the last compute in forward, or it is not a pure norm.
|
||||
if _has_trailing_compute(graph, tail):
|
||||
return None
|
||||
eps_attr, eps = cls._eps_source(graph, module)
|
||||
return cls(
|
||||
zero_centered=zero_centered,
|
||||
source_cls=type(module).__name__,
|
||||
eps_attr=eps_attr,
|
||||
eps=eps,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def _eps_source(
|
||||
cls, graph: fx.Graph, module: nn.Module
|
||||
) -> tuple[str | None, float | None]:
|
||||
"""Where `fuse` should read `eps` from, resolved once per class."""
|
||||
eps = cls._eps_from_graph(graph)
|
||||
if eps is None:
|
||||
return None, None
|
||||
# Whatever supplied the constant must still equal it.
|
||||
candidates = {
|
||||
name: value
|
||||
for name, value in vars(module).items()
|
||||
if isinstance(value, float) and value == eps
|
||||
}
|
||||
# Use unique markers and retrace to verify exactly which attribute is eps.
|
||||
markers = {float(-index - 1): name for index, name in enumerate(candidates)}
|
||||
marked = None
|
||||
if markers:
|
||||
try:
|
||||
for marker, name in markers.items():
|
||||
setattr(module, name, marker)
|
||||
if (remarked := trace(module)) is not None:
|
||||
marked = cls._eps_from_graph(remarked)
|
||||
finally:
|
||||
for name, value in candidates.items():
|
||||
setattr(module, name, value)
|
||||
if marked is not None and (name := markers.get(marked)) is not None:
|
||||
return name, None
|
||||
logger.debug_once(
|
||||
"%s does not hold its eps (%s) in an attribute. Every instance in this "
|
||||
"model will use the value traced from this instance. If this is not "
|
||||
"desired, consider storing and reading eps using attribute of %s.",
|
||||
type(module).__name__,
|
||||
eps,
|
||||
type(module).__name__,
|
||||
)
|
||||
return None, eps
|
||||
|
||||
@staticmethod
|
||||
def _eps_from_graph(graph: fx.Graph) -> float | None:
|
||||
"""Extract the `eps` constant from the graph, if present."""
|
||||
if (x := find_node(graph, lambda n: n.op == "placeholder")) is None:
|
||||
return None
|
||||
fused = find_node(graph, lambda n: is_op(n, "rms_norm"))
|
||||
if fused is not None and peel(_operand(fused, 0, "input")) is x:
|
||||
args, kwargs = fused.args, fused.kwargs
|
||||
eps = args[3] if len(args) > 3 else kwargs.get("eps")
|
||||
return float(eps) if isinstance(eps, (int, float)) else None
|
||||
for node in graph.nodes:
|
||||
if _is_inverse_sqrt(node) and (eps := _variance_eps(node, x)) is not None:
|
||||
return eps
|
||||
return None
|
||||
|
||||
def validate(self, module: nn.Module, vllm_config: "VllmConfig") -> bool:
|
||||
return True
|
||||
|
||||
def fuse(
|
||||
self, module: nn.Module, prefix: str, vllm_config: "VllmConfig"
|
||||
) -> nn.Module:
|
||||
"""Fuse the matched RMSNorm pattern into a vLLM fused RMSNorm CustomOp."""
|
||||
weight = getattr(module, "weight", None)
|
||||
hidden_size = weight.size(0) if weight is not None else 0
|
||||
eps = getattr(module, self.eps_attr, None) if self.eps_attr else self.eps
|
||||
if not isinstance(eps, (int, float)):
|
||||
# If eps was not detected, match torch behaviour.
|
||||
config_dtype = vllm_config.model_config.dtype
|
||||
dtype = weight.dtype if weight is not None else config_dtype
|
||||
eps = torch.finfo(dtype).eps
|
||||
if self.zero_centered:
|
||||
return TPAwareGemmaRMSNorm(hidden_size=hidden_size, eps=eps)
|
||||
return TPAwareRMSNorm(
|
||||
hidden_size=hidden_size,
|
||||
eps=eps,
|
||||
has_weight=weight is not None,
|
||||
dtype=weight.dtype if weight is not None else None,
|
||||
)
|
||||
@@ -1,818 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""fx tracing and forward-source rewriting for the Transformers backend fusers.
|
||||
|
||||
A small engine, independent of any particular pattern: trace a module's forward
|
||||
with `torch.fx` (tolerating a partial graph), inspect the resulting nodes, and
|
||||
rewrite the forward's *source* (AST) so only matched calls change while the rest
|
||||
stays live Python. `fusion.py` builds the concrete fusion patterns on top.
|
||||
"""
|
||||
|
||||
import ast
|
||||
import contextlib
|
||||
import inspect
|
||||
import operator
|
||||
import textwrap
|
||||
from collections.abc import Callable
|
||||
from itertools import chain
|
||||
from typing import Any
|
||||
from unittest import mock
|
||||
|
||||
import torch
|
||||
from torch import fx, nn
|
||||
from torch.nn import functional as F
|
||||
from typing_extensions import TypeIs
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_UNKNOWN: Any = object()
|
||||
"""Sentinel meta value for proxies whose concrete value could not be inferred.
|
||||
Distinct from `None`, which is a valid concrete value (e.g. `attn_weights`)."""
|
||||
|
||||
_MODULE_CALL = nn.Module.__call__
|
||||
"""The unpatched `nn.Module.__call__`. During tracing fx patches it to record
|
||||
`call_module` nodes; meta execution must call modules for real."""
|
||||
|
||||
_MODULE_GETATTR = nn.Module.__getattr__
|
||||
"""The unpatched `nn.Module.__getattr__`. During tracing fx patches it to record
|
||||
`get_attr` nodes; reading one while inferring a `get_attr`'s own value would
|
||||
recurse until the stack runs out, truncating the graph."""
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def _unpatched_modules():
|
||||
"""Restores the module hooks fx patches, so meta execution records nothing."""
|
||||
with (
|
||||
mock.patch.object(nn.Module, "__call__", _MODULE_CALL),
|
||||
mock.patch.object(nn.Module, "__getattr__", _MODULE_GETATTR),
|
||||
):
|
||||
yield
|
||||
|
||||
|
||||
def is_leaf_call(node: object) -> TypeIs[fx.Node]:
|
||||
"""Is node a call recorded by `_as_leaf_call` (e.g. an attention interface)."""
|
||||
return isinstance(node, fx.Node) and node.meta.get("leaf_call", False)
|
||||
|
||||
|
||||
def _reference_weight(module: nn.Module) -> torch.Tensor | None:
|
||||
"""A weight whose trailing dim is the module's hidden size.
|
||||
|
||||
Linears and 2-D gate weights are `[out, hidden]`; norm weights are
|
||||
`[hidden]`. Used to fabricate a placeholder input of matching size/dtype."""
|
||||
for child in module.modules():
|
||||
if isinstance(child, nn.Linear):
|
||||
return child.weight
|
||||
for param in module.parameters():
|
||||
if param.ndim in (1, 2):
|
||||
return param
|
||||
return None
|
||||
|
||||
|
||||
class _MetaProxy(fx.Proxy):
|
||||
"""Proxy carrying the meta-tensor value of the traced expression.
|
||||
|
||||
Shape questions (`len`, iteration, `.shape` unpacks) are answered by
|
||||
executing each op on the meta values, so PyTorch's meta kernels are the
|
||||
single source of shape inference — no per-op rules."""
|
||||
|
||||
meta: Any = _UNKNOWN
|
||||
|
||||
def __len__(self) -> int:
|
||||
if self.meta is not _UNKNOWN:
|
||||
return len(self.meta)
|
||||
return super().__len__() # type: ignore[misc]
|
||||
|
||||
def __getattr__(self, k: str) -> "_MetaAttribute":
|
||||
return _MetaAttribute(self, k)
|
||||
|
||||
def __setitem__(self, key: object, value: object) -> None:
|
||||
# Record `obj[key] = value` so tracing continues past in-place writes
|
||||
# into containers (e.g. Gemma 4's `shared_kv_states[...] = ...`).
|
||||
self.tracer.create_proxy(
|
||||
"call_function", operator.setitem, (self, key, value), {}
|
||||
)
|
||||
|
||||
|
||||
class _MetaAttribute(_MetaProxy, fx.proxy.Attribute):
|
||||
"""Attribute proxy (e.g. `x.shape`) carrying its meta value.
|
||||
|
||||
`Proxy.__getattr__` constructs `Attribute` directly, bypassing
|
||||
`Tracer.proxy`, so the meta value must be grafted on here too."""
|
||||
|
||||
def __init__(self, root: fx.Proxy, attr: str):
|
||||
super().__init__(root, attr)
|
||||
root_meta = getattr(root, "meta", _UNKNOWN)
|
||||
if root_meta is not _UNKNOWN:
|
||||
with contextlib.suppress(Exception):
|
||||
self.meta = getattr(root_meta, attr)
|
||||
|
||||
|
||||
class _AllLeafTracer(fx.Tracer):
|
||||
"""Tracer that treats every submodule as a leaf.
|
||||
|
||||
Each child stays one `call_module` node, so matching sees the module's own
|
||||
forward structure (activations aren't decomposed into e.g. `sigmoid * x`).
|
||||
Every traced op is also executed on meta tensors (see `_MetaProxy`) so
|
||||
shape unpacks and `*`-splats trace through; anything else untraceable ends
|
||||
the trace early and the partial graph is matched.
|
||||
"""
|
||||
|
||||
varkw: str | None = None
|
||||
"""Name of the traced forward's `**kwargs` parameter, if any."""
|
||||
|
||||
def is_leaf_module(self, m: nn.Module, module_qualified_name: str) -> bool:
|
||||
return True
|
||||
|
||||
def proxy(self, node: fx.Node) -> fx.Proxy:
|
||||
return _MetaProxy(node, self)
|
||||
|
||||
def create_proxy(self, kind, target, args, kwargs, *extra, **extra_kwargs):
|
||||
proxy = super().create_proxy(kind, target, args, kwargs, *extra, **extra_kwargs)
|
||||
if isinstance(proxy, _MetaProxy) and proxy.meta is _UNKNOWN:
|
||||
# A failure stays _UNKNOWN; only fatal if a shape question is asked.
|
||||
with contextlib.suppress(Exception):
|
||||
proxy.meta = self._infer_meta(kind, target, args, kwargs)
|
||||
return proxy
|
||||
|
||||
def _infer_meta(self, kind: str, target: object, args: tuple, kwargs: dict):
|
||||
"""Execute the op on meta tensors; PyTorch infers the output value."""
|
||||
if kind == "placeholder":
|
||||
# vLLM always feeds the model [1, seq_len, hidden_size] hidden states.
|
||||
weight = _reference_weight(self.root)
|
||||
if str(target) == "hidden_states" and weight is not None:
|
||||
return torch.empty(
|
||||
1, 8, weight.shape[-1], dtype=weight.dtype, device="meta"
|
||||
)
|
||||
return _UNKNOWN
|
||||
if kind == "get_attr":
|
||||
with _unpatched_modules():
|
||||
value = operator.attrgetter(str(target))(self.root)
|
||||
if isinstance(value, torch.Tensor):
|
||||
value = torch.empty_like(value, device="meta")
|
||||
return value
|
||||
unknown = False
|
||||
|
||||
def meta_of(arg: object) -> object:
|
||||
nonlocal unknown
|
||||
if isinstance(arg, fx.Proxy):
|
||||
meta = getattr(arg, "meta", _UNKNOWN)
|
||||
unknown = unknown or meta is _UNKNOWN
|
||||
return meta
|
||||
return arg
|
||||
|
||||
meta_args = fx.node.map_aggregate(args, meta_of)
|
||||
meta_kwargs = fx.node.map_aggregate(kwargs, meta_of)
|
||||
if unknown:
|
||||
return _UNKNOWN
|
||||
if kind == "call_function":
|
||||
assert callable(target)
|
||||
return target(*meta_args, **meta_kwargs)
|
||||
if kind == "call_method":
|
||||
receiver, *rest = meta_args
|
||||
return getattr(receiver, str(target))(*rest, **meta_kwargs)
|
||||
if kind == "call_module":
|
||||
# Run the child's forward with all its state on "meta", without
|
||||
# mutating it (at match time params may be meta but buffers real).
|
||||
child = self.root.get_submodule(str(target))
|
||||
state = {
|
||||
name: torch.empty_like(tensor, device="meta")
|
||||
for name, tensor in chain(
|
||||
child.named_parameters(), child.named_buffers()
|
||||
)
|
||||
}
|
||||
with _unpatched_modules():
|
||||
return torch.func.functional_call(child, state, meta_args, meta_kwargs)
|
||||
return _UNKNOWN
|
||||
|
||||
def _is_varkw(self, node: object) -> bool:
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "placeholder"
|
||||
and str(node.target).lstrip("*") == self.varkw
|
||||
)
|
||||
|
||||
def iter(self, obj: fx.Proxy):
|
||||
# Assume kwargs is always empty to simplify tracing.
|
||||
node = obj.node
|
||||
if self._is_varkw(node) or (
|
||||
node.op == "call_method"
|
||||
and node.target == "keys"
|
||||
and self._is_varkw(node.args[0])
|
||||
):
|
||||
return iter(())
|
||||
meta = getattr(obj, "meta", _UNKNOWN)
|
||||
if meta is _UNKNOWN:
|
||||
return super().iter(obj)
|
||||
return iter([obj[i] for i in range(len(meta))])
|
||||
|
||||
|
||||
def _as_leaf_call(fn: Callable, length: int | None = None) -> Callable:
|
||||
"""Wrap any callable so tracing records it as one opaque `call_function` node.
|
||||
|
||||
Lets the trace continue past untraceable bodies. Only the proxy arguments carry into
|
||||
the node's dataflow; the rest are dropped rather than lifted into the graph.
|
||||
`length` declares how many values the callable returns, so unpacking its result also
|
||||
traces. Called without proxies (i.e. outside tracing), the wrapper is a passthrough.
|
||||
"""
|
||||
|
||||
def leaf(*args, **kwargs):
|
||||
proxies = tuple(arg for arg in args if isinstance(arg, fx.Proxy))
|
||||
if not proxies:
|
||||
return fn(*args, **kwargs)
|
||||
proxy = proxies[0].tracer.create_proxy("call_function", fn, proxies, {})
|
||||
proxy.node.meta["leaf_call"] = True
|
||||
if length is not None:
|
||||
# The body never executes, so fabricate a value of the declared length.
|
||||
proxy.meta = (_UNKNOWN,) * length
|
||||
return proxy
|
||||
|
||||
return leaf
|
||||
|
||||
|
||||
def _leaf_attention_interfaces():
|
||||
"""Patch `AttentionInterface.get_interface` so traced forwards see a leaf node.
|
||||
|
||||
`vllm_attention_function` needs runtime context so it is untraceable.
|
||||
Every interface returns `(attn_output, attn_weights)`."""
|
||||
from transformers.modeling_utils import AttentionInterface
|
||||
|
||||
original = AttentionInterface.get_interface
|
||||
|
||||
def get_interface(self, *args, **kwargs):
|
||||
return _as_leaf_call(original(self, *args, **kwargs), length=2)
|
||||
|
||||
return mock.patch.object(AttentionInterface, "get_interface", get_interface)
|
||||
|
||||
|
||||
def trace(module: nn.Module) -> fx.Graph | None:
|
||||
"""Trace `module.forward`, returning the partial graph on failure."""
|
||||
parameters = forward_parameters(type(module))
|
||||
# vLLM never passes `past_key_values` so it is always the default value of `None`.
|
||||
# Make this concrete to simplify tracing.
|
||||
concrete_args = None
|
||||
if "past_key_values" in parameters:
|
||||
concrete_args = {"past_key_values": None}
|
||||
# Get the name of the kwargs parameter passed to module.forward (usually "kwargs")
|
||||
tracer = _AllLeafTracer()
|
||||
tracer.varkw = next(
|
||||
(
|
||||
p.name
|
||||
for p in parameters.values()
|
||||
if p.kind is inspect.Parameter.VAR_KEYWORD
|
||||
),
|
||||
None,
|
||||
)
|
||||
try:
|
||||
with _leaf_attention_interfaces():
|
||||
return tracer.trace(module, concrete_args=concrete_args)
|
||||
except Exception as exc:
|
||||
logger.debug("Could not fully trace %s: %s", type(module), exc)
|
||||
return getattr(tracer, "graph", None)
|
||||
|
||||
|
||||
def recover_forward(cls: type[nn.Module]) -> tuple[ast.FunctionDef, Callable]:
|
||||
"""Parse the source of `cls.forward`, ready for rewriting."""
|
||||
fn = inspect.unwrap(cls.forward)
|
||||
if fn.__code__.co_freevars:
|
||||
raise ValueError("forward is a closure")
|
||||
tree = ast.parse(textwrap.dedent(inspect.getsource(fn)))
|
||||
funcdef = tree.body[0]
|
||||
if not isinstance(funcdef, ast.FunctionDef):
|
||||
raise ValueError("source is not a plain function definition")
|
||||
# `fn` is already unwrapped; don't re-apply its decorators
|
||||
funcdef.decorator_list.clear()
|
||||
# Annotations may not evaluate outside the defining module (e.g. with
|
||||
# postponed evaluation); they're not needed at runtime
|
||||
funcdef.returns = None
|
||||
args = funcdef.args
|
||||
for arg in (
|
||||
*args.posonlyargs,
|
||||
*args.args,
|
||||
*args.kwonlyargs,
|
||||
*filter(None, (args.vararg, args.kwarg)),
|
||||
):
|
||||
arg.annotation = None
|
||||
# Recompiling outside the class body would break name mangling
|
||||
for node in ast.walk(funcdef):
|
||||
name = getattr(node, "attr", None) or getattr(node, "id", None)
|
||||
if name and name.startswith("__") and not name.endswith("__"):
|
||||
raise ValueError(f"{name} would be name mangled")
|
||||
return funcdef, fn
|
||||
|
||||
|
||||
def forward_parameters(cls: type[nn.Module]) -> dict[str, inspect.Parameter]:
|
||||
"""`cls.forward`'s signature parameters, or empty if uninspectable."""
|
||||
try:
|
||||
return dict(inspect.signature(cls.forward).parameters)
|
||||
except (TypeError, ValueError):
|
||||
return {}
|
||||
|
||||
|
||||
def forward_input_count(cls: type[nn.Module]) -> int:
|
||||
"""The number of tensor inputs `cls.forward` declares, excluding `self` and
|
||||
any `*args`/`**kwargs`. Read from the signature, so it is independent of
|
||||
whether the trace completes (unlike counting placeholders)."""
|
||||
params = list(forward_parameters(cls).values())
|
||||
if not params:
|
||||
return 1 # uninspectable: assume a single input and let matching decide
|
||||
fixed = (
|
||||
inspect.Parameter.POSITIONAL_ONLY,
|
||||
inspect.Parameter.POSITIONAL_OR_KEYWORD,
|
||||
inspect.Parameter.KEYWORD_ONLY,
|
||||
)
|
||||
return sum(1 for p in params[1:] if p.kind in fixed)
|
||||
|
||||
|
||||
def compile_forward(funcdef: ast.FunctionDef, fn: Callable) -> Callable:
|
||||
"""Compile `funcdef` in `fn`'s module so tracebacks point at the source."""
|
||||
module = ast.Module(body=[funcdef], type_ignores=[])
|
||||
ast.fix_missing_locations(module)
|
||||
ast.increment_lineno(module, fn.__code__.co_firstlineno - 1)
|
||||
code = compile(module, fn.__code__.co_filename, "exec")
|
||||
namespace: dict = {}
|
||||
exec(code, fn.__globals__, namespace)
|
||||
return namespace[funcdef.name]
|
||||
|
||||
|
||||
def self_call_and_refs(
|
||||
funcdef: ast.FunctionDef, name: str
|
||||
) -> tuple[ast.Call, list[ast.Attribute]]:
|
||||
"""The unique `self.<name>(arg)` call, plus every other `self.<name>` reference.
|
||||
|
||||
Raises unless exactly one single-argument `self.<name>(arg)` call exists, so
|
||||
one fx `call_module` node maps to one syntactic call site. The remaining
|
||||
references (e.g. `is not None` guards) are returned for the caller to resolve.
|
||||
"""
|
||||
uses = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Attribute)
|
||||
and node.attr == name
|
||||
and isinstance(node.value, ast.Name)
|
||||
and node.value.id == "self"
|
||||
]
|
||||
calls = [
|
||||
node
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, ast.Call)
|
||||
and any(node.func is use for use in uses)
|
||||
and len(node.args) == 1
|
||||
and not isinstance(node.args[0], ast.Starred)
|
||||
and not node.keywords
|
||||
]
|
||||
if len(calls) != 1:
|
||||
raise ValueError(f"{name} is not called exactly once as self.{name}(arg)")
|
||||
call = calls[0]
|
||||
other_refs = [use for use in uses if use is not call.func]
|
||||
return call, other_refs
|
||||
|
||||
|
||||
def single_self_call(funcdef: ast.FunctionDef, name: str) -> ast.Call:
|
||||
"""The unique `self.<name>(arg)` call in `funcdef`.
|
||||
|
||||
Raises unless `name` appears exactly once, as such a call, so the source
|
||||
rewrite agrees with the fx match.
|
||||
"""
|
||||
call, other_refs = self_call_and_refs(funcdef, name)
|
||||
if other_refs:
|
||||
raise ValueError(f"{name} is referenced {len(other_refs) + 1} times")
|
||||
return call
|
||||
|
||||
|
||||
def _is_none(node: ast.expr) -> bool:
|
||||
return isinstance(node, ast.Constant) and node.value is None
|
||||
|
||||
|
||||
def _in_boolean_context(funcdef: ast.FunctionDef, ref: ast.expr) -> bool:
|
||||
"""Is `ref` used only for its truth value (a test, or a `not` operand)?
|
||||
|
||||
In these positions the object's identity never escapes, so a reference that
|
||||
is always truthy can be replaced by `True`. `and`/`or` are excluded: they
|
||||
yield an operand, so the module could escape (`x and self.<name>`)."""
|
||||
for node in ast.walk(funcdef):
|
||||
if (
|
||||
isinstance(node, (ast.If, ast.IfExp, ast.While, ast.Assert))
|
||||
and node.test is ref
|
||||
):
|
||||
return True
|
||||
if (
|
||||
isinstance(node, ast.UnaryOp)
|
||||
and isinstance(node.op, ast.Not)
|
||||
and node.operand is ref
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def bypass_existence_guard(
|
||||
funcdef: ast.FunctionDef, ref: ast.Attribute, name: str
|
||||
) -> None:
|
||||
"""Fold a guard on `self.<name>`'s existence to its constant value.
|
||||
|
||||
The fuser deletes `self.<name>` and binds only instances where it exists as a
|
||||
truthy `nn.Linear` (guaranteed by the match, the cache key, and `validate`),
|
||||
so a guard testing its presence is a fusion invariant. Two forms are folded:
|
||||
|
||||
- an identity `None` check `self.<name> is None` -> `False`,
|
||||
`self.<name> is not None` -> `True` (either operand order). `==`/`!=` are
|
||||
*not* folded: `is_linear` accepts `nn.Linear` subclasses, which may override
|
||||
`__eq__`/`__ne__`, so equality is not guaranteed to track identity.
|
||||
- a bare truthiness test (`if self.<name>:`, `... if self.<name> else ...`,
|
||||
`not self.<name>`): the reference itself to `True`.
|
||||
|
||||
Any other surviving reference escapes the projection's value, which no longer
|
||||
exists after fusion, so refuse rather than change semantics."""
|
||||
for node in ast.walk(funcdef):
|
||||
# `self.<name> is (not) None`, either operand order.
|
||||
if not (isinstance(node, ast.Compare) and len(node.ops) == 1):
|
||||
continue
|
||||
(op,), (right,) = node.ops, node.comparators
|
||||
if not isinstance(op, (ast.Is, ast.IsNot)):
|
||||
continue
|
||||
if (ref is node.left and _is_none(right)) or (
|
||||
ref is right and _is_none(node.left)
|
||||
):
|
||||
value = isinstance(op, ast.IsNot)
|
||||
replace_expr(funcdef, node, ast.copy_location(ast.Constant(value), node))
|
||||
return
|
||||
# A bare truthiness test (statement `if`, ternary, or `not`); an `nn.Linear`
|
||||
# is always truthy, so the reference folds to `True`.
|
||||
if _in_boolean_context(funcdef, ref):
|
||||
replace_expr(funcdef, ref, ast.copy_location(ast.Constant(True), ref))
|
||||
return
|
||||
raise ValueError(f"{name} is referenced outside an existence guard")
|
||||
|
||||
|
||||
def block_chain(
|
||||
block: list[ast.stmt], node: ast.AST
|
||||
) -> list[tuple[list[ast.stmt], int]]:
|
||||
"""Path of (statement list, index) pairs from `block` down to `node`.
|
||||
|
||||
Each pair names a nested block and the index in it of the statement
|
||||
containing `node`; the last pair is `node`'s innermost block. Empty if
|
||||
`node` is not in `block`. The longest common prefix of several nodes' chains
|
||||
is the innermost block that dominates them all.
|
||||
"""
|
||||
for index, stmt in enumerate(block):
|
||||
if not any(child is node for child in ast.walk(stmt)):
|
||||
continue
|
||||
child_blocks = [
|
||||
getattr(stmt, fld, None) for fld in ("body", "orelse", "finalbody")
|
||||
]
|
||||
child_blocks += [h.body for h in getattr(stmt, "handlers", [])]
|
||||
child_blocks += [c.body for c in getattr(stmt, "cases", [])]
|
||||
for child_block in child_blocks:
|
||||
if (
|
||||
isinstance(child_block, list)
|
||||
and child_block
|
||||
and (tail := block_chain(child_block, node))
|
||||
):
|
||||
return [(block, index), *tail]
|
||||
return [(block, index)]
|
||||
return []
|
||||
|
||||
|
||||
def _base_name(node: ast.expr) -> str | None:
|
||||
"""The root `Name` of an attribute/subscript chain (`a.b[c]` -> `a`)."""
|
||||
while isinstance(node, (ast.Attribute, ast.Subscript)):
|
||||
node = node.value
|
||||
return node.id if isinstance(node, ast.Name) else None
|
||||
|
||||
|
||||
def _rebound_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names rebound outright within `region` (`x = ...`, `del x`).
|
||||
|
||||
These are `Name` nodes in a `Store`/`Del` context."""
|
||||
return {
|
||||
node.id
|
||||
for stmt in region
|
||||
for node in ast.walk(stmt)
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, (ast.Store, ast.Del))
|
||||
}
|
||||
|
||||
|
||||
def _inplace_target(node: ast.AST) -> str | None:
|
||||
"""Base name `node` mutates in place, if any.
|
||||
|
||||
A write through the name (`x[i] = ...`, `x.attr = ...`) or an in-place method
|
||||
call (`x.mul_(...)`) leaves the base name in a `Load` context, so it is not a
|
||||
plain `Name` store (see `_rebound_names`)."""
|
||||
if isinstance(node, (ast.Attribute, ast.Subscript)) and isinstance(
|
||||
node.ctx, (ast.Store, ast.Del)
|
||||
):
|
||||
return _base_name(node)
|
||||
if (
|
||||
isinstance(node, ast.Call)
|
||||
and isinstance(node.func, ast.Attribute)
|
||||
and node.func.attr.endswith("_")
|
||||
and not node.func.attr.endswith("__")
|
||||
):
|
||||
return _base_name(node.func.value)
|
||||
return None
|
||||
|
||||
|
||||
def _mutated_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names mutated in place within `region` (see `_inplace_target`)."""
|
||||
return {
|
||||
base
|
||||
for stmt in region
|
||||
for node in ast.walk(stmt)
|
||||
if (base := _inplace_target(node)) is not None
|
||||
}
|
||||
|
||||
|
||||
def _call_name(call: ast.Call) -> str:
|
||||
"""The called function's own name, without its qualifier."""
|
||||
if isinstance(call.func, ast.Attribute):
|
||||
return call.func.attr
|
||||
return call.func.id if isinstance(call.func, ast.Name) else ""
|
||||
|
||||
|
||||
def _arg_names(call: ast.Call) -> set[str]:
|
||||
"""Every name `call` reads through its arguments."""
|
||||
reads = [*call.args, *(kw.value for kw in call.keywords)]
|
||||
return {
|
||||
node.id
|
||||
for arg in reads
|
||||
for node in ast.walk(arg)
|
||||
if isinstance(node, ast.Name) and isinstance(node.ctx, ast.Load)
|
||||
}
|
||||
|
||||
|
||||
def _writes_through_args(call: ast.Call) -> bool:
|
||||
"""Does `call` write through an argument rather than return a new value?
|
||||
|
||||
Torch spells this three ways: an `out=` destination, `inplace=True`, and the
|
||||
trailing-underscore convention (`torch.relu_(x)`). The underscore is checked
|
||||
on the function's own name, so it catches the free-function form that
|
||||
`_inplace_target` cannot (there the base name is the module, `torch`).
|
||||
"""
|
||||
for keyword in call.keywords:
|
||||
if keyword.arg == "out":
|
||||
return True
|
||||
if keyword.arg == "inplace" and not (
|
||||
isinstance(keyword.value, ast.Constant) and keyword.value.value is False
|
||||
):
|
||||
return True
|
||||
name = _call_name(call)
|
||||
return name.endswith("_") and not name.endswith("__")
|
||||
|
||||
|
||||
def written_names(region: list[ast.stmt]) -> set[str]:
|
||||
"""Names `region` may rebind or write through.
|
||||
|
||||
Beyond plain rebinding and writes through the name itself (`_rebound_names`,
|
||||
`_mutated_names`), a call can write through an argument without naming it as
|
||||
a target: `F.relu(x, inplace=True)`, `torch.add(x, y, out=x)`,
|
||||
`torch.relu_(x)`. A statement-level call whose result is discarded is treated
|
||||
the same way, since it can only be there for a side effect.
|
||||
"""
|
||||
written = _rebound_names(region) | _mutated_names(region)
|
||||
for stmt in region:
|
||||
if isinstance(stmt, ast.Expr) and isinstance(stmt.value, ast.Call):
|
||||
written |= _arg_names(stmt.value)
|
||||
for node in ast.walk(stmt):
|
||||
if isinstance(node, ast.Call) and _writes_through_args(node):
|
||||
written |= _arg_names(node)
|
||||
return written
|
||||
|
||||
|
||||
_METADATA_ATTRS = frozenset(
|
||||
{"shape", "dtype", "device", "ndim", "size", "dim", "numel", "item"}
|
||||
)
|
||||
|
||||
|
||||
def _aliasing_reads(node: ast.expr) -> set[str]:
|
||||
"""Names `node` reads in a way that could yield a view of them.
|
||||
|
||||
Metadata (`x.shape`, `x.size(0)`, `x.dtype`) is ints and tuples sharing no
|
||||
storage with `x`, so reaching a name only through metadata cannot alias it.
|
||||
Shape arithmetic is pervasive between projections, so propagating through it
|
||||
would taint nearly every later name.
|
||||
"""
|
||||
names: set[str] = set()
|
||||
stack: list[ast.AST] = [node]
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if isinstance(current, ast.Attribute) and current.attr in _METADATA_ATTRS:
|
||||
continue
|
||||
if (
|
||||
isinstance(current, ast.Call)
|
||||
and isinstance(current.func, ast.Attribute)
|
||||
and current.func.attr in _METADATA_ATTRS
|
||||
):
|
||||
stack.extend(current.args)
|
||||
stack.extend(keyword.value for keyword in current.keywords)
|
||||
continue
|
||||
if isinstance(current, ast.Name) and isinstance(current.ctx, ast.Load):
|
||||
names.add(current.id)
|
||||
stack.extend(ast.iter_child_nodes(current))
|
||||
return names
|
||||
|
||||
|
||||
def _returns_fresh(node: ast.expr, module: nn.Module) -> bool:
|
||||
"""Does evaluating `node` allocate, rather than return a view of its input?
|
||||
|
||||
A `nn.Linear` writes its output into new storage, so its result aliases
|
||||
nothing that went in, and a chain of view ops on top of it aliases only that
|
||||
fresh tensor. Walking the receiver chain down to a `self.<linear>(...)` call
|
||||
therefore proves the value cannot alias the projection input. A literal
|
||||
aliases nothing either, and a branch or sequence is fresh when every part is
|
||||
-- which is how a guarded projection (`p(x) if p is not None else None`)
|
||||
stays untracked. A call on an attribute that is missing or `None` cannot run
|
||||
at all, so a branch selecting a different configuration is fresh too.
|
||||
"""
|
||||
if isinstance(node, ast.Constant):
|
||||
return True
|
||||
if isinstance(node, ast.IfExp):
|
||||
if isinstance(node.test, ast.Constant):
|
||||
# A folded existence guard leaves the untaken branch as dead code.
|
||||
taken = node.body if node.test.value else node.orelse
|
||||
return _returns_fresh(taken, module)
|
||||
return _returns_fresh(node.body, module) and _returns_fresh(node.orelse, module)
|
||||
if isinstance(node, (ast.Tuple, ast.List)):
|
||||
return all(_returns_fresh(elt, module) for elt in node.elts)
|
||||
while isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute):
|
||||
func = node.func
|
||||
if isinstance(func.value, ast.Name) and func.value.id == "self":
|
||||
attribute = getattr(module, func.attr, None)
|
||||
if attribute is None or isinstance(attribute, nn.Linear):
|
||||
# A missing or `None` attribute cannot be called, so the branch
|
||||
# selecting it never runs (`q_proj` on a q-LoRA layer).
|
||||
return True
|
||||
# Anything else can only return storage it was handed, so it is
|
||||
# fresh exactly when its inputs are (`q_norm(q_proj(x))`).
|
||||
return bool(node.args) and all(
|
||||
_returns_fresh(arg, module) for arg in node.args
|
||||
)
|
||||
node = func.value
|
||||
return False
|
||||
|
||||
|
||||
def aliasing_names(
|
||||
funcdef: ast.FunctionDef, seed: set[str], module: nn.Module
|
||||
) -> set[str]:
|
||||
"""`seed` plus every name that may alias one of them.
|
||||
|
||||
A view shares storage with its base (`h = x.view(-1)`), so mutating `h`
|
||||
mutates `x`; tracking only `seed` would miss that. Any assignment reading a
|
||||
tracked name therefore taints its targets, transitively, unless its value is
|
||||
freshly allocated (`_returns_fresh`).
|
||||
"""
|
||||
names = set(seed)
|
||||
assigns = [
|
||||
(node, node.value)
|
||||
for node in ast.walk(funcdef)
|
||||
if isinstance(node, (ast.Assign, ast.AnnAssign))
|
||||
and node.value is not None
|
||||
and not _returns_fresh(node.value, module)
|
||||
]
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for node, value in assigns:
|
||||
if not _aliasing_reads(value) & names:
|
||||
continue
|
||||
targets = node.targets if isinstance(node, ast.Assign) else [node.target]
|
||||
stores = {
|
||||
child.id
|
||||
for target in targets
|
||||
for child in ast.walk(target)
|
||||
if isinstance(child, ast.Name) and isinstance(child.ctx, ast.Store)
|
||||
}
|
||||
if stores - names:
|
||||
names |= stores
|
||||
changed = True
|
||||
return names
|
||||
|
||||
|
||||
def replace_expr(module: ast.AST, old: ast.expr, new: ast.expr) -> None:
|
||||
"""Replace the expression `old` (by identity) with `new` within `module`."""
|
||||
|
||||
class _Replacer(ast.NodeTransformer):
|
||||
def visit(self, node: ast.AST) -> ast.AST:
|
||||
if node is old:
|
||||
return new
|
||||
return super().generic_visit(node)
|
||||
|
||||
_Replacer().visit(module)
|
||||
|
||||
|
||||
def is_complete(graph: fx.Graph) -> bool:
|
||||
"""Did the trace run to completion? A partial graph has no `output` node.
|
||||
|
||||
Only a complete graph can be read for the *absence* of an op; in a partial
|
||||
one the tail is exactly what is missing."""
|
||||
return any(node.op == "output" for node in graph.nodes)
|
||||
|
||||
|
||||
def find_node(graph: fx.Graph, predicate: Callable[[fx.Node], bool]) -> fx.Node | None:
|
||||
"""The first node in `graph` matching `predicate`, or `None`."""
|
||||
return next((n for n in graph.nodes if predicate(n)), None)
|
||||
|
||||
|
||||
def output_value(graph: fx.Graph) -> object | None:
|
||||
"""The value the graph's `output` node returns, if the trace reached one."""
|
||||
output = find_node(graph, lambda n: n.op == "output")
|
||||
if output is None or not output.args:
|
||||
return None
|
||||
return output.args[0]
|
||||
|
||||
|
||||
def upstream_linear(node: object, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear producing `node`, walking back through splits/reshapes.
|
||||
|
||||
Non-linear submodules are transparent too (e.g. the dropout GPT-style
|
||||
attentions apply after their output projection). Never walks through a leaf
|
||||
call (e.g. an attention interface): its inputs are what attention consumes,
|
||||
not what produced the value."""
|
||||
stack = [node]
|
||||
seen: set[fx.Node] = set()
|
||||
while stack:
|
||||
current = stack.pop()
|
||||
if not isinstance(current, fx.Node) or current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
if is_linear(current, module):
|
||||
return current
|
||||
if current.op in (
|
||||
"call_function",
|
||||
"call_method",
|
||||
"call_module",
|
||||
) and not is_leaf_call(current):
|
||||
stack.extend(current.args)
|
||||
return None
|
||||
|
||||
|
||||
def downstream_linear(node: fx.Node, module: nn.Module) -> fx.Node | None:
|
||||
"""Nearest linear consuming `node`'s output, walking through casts/scalings.
|
||||
|
||||
Never walks through a leaf call (e.g. an attention interface): what crosses
|
||||
it is consumed by the attention computation, not projected."""
|
||||
queue = list(node.users)
|
||||
seen: set[fx.Node] = set()
|
||||
while queue:
|
||||
current = queue.pop(0)
|
||||
if current in seen:
|
||||
continue
|
||||
seen.add(current)
|
||||
if is_linear(current, module):
|
||||
return current
|
||||
if current.op in ("call_function", "call_method") and not is_leaf_call(current):
|
||||
queue.extend(current.users)
|
||||
return None
|
||||
|
||||
|
||||
def returned_linear(graph: fx.Graph, module: nn.Module) -> str | None:
|
||||
"""Name of the Linear producing the graph's (first) output value."""
|
||||
value = output_value(graph)
|
||||
if isinstance(value, (tuple, list)) and value:
|
||||
value = value[0]
|
||||
linear = upstream_linear(value, module)
|
||||
return None if linear is None else str(linear.target)
|
||||
|
||||
|
||||
def is_linear(node: fx.Node, module: nn.Module) -> bool:
|
||||
"""Is node `nn.Linear.__call__()`."""
|
||||
return node.op == "call_module" and isinstance(
|
||||
module.get_submodule(node.target), nn.Linear
|
||||
)
|
||||
|
||||
|
||||
_DTYPE_CASTS = frozenset({"to", "float", "double", "half", "bfloat16", "type_as"})
|
||||
|
||||
|
||||
def peel(node: object) -> object:
|
||||
"""Strip dtype-cast wrappers (`.to(...)`, `.float()`, `.type_as(...)`)."""
|
||||
while (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_method"
|
||||
and node.target in _DTYPE_CASTS
|
||||
):
|
||||
node = node.args[0]
|
||||
return node
|
||||
|
||||
|
||||
def is_fn(node: object, target: object) -> TypeIs[fx.Node]:
|
||||
"""Is node `<target>()`."""
|
||||
return (
|
||||
isinstance(node, fx.Node)
|
||||
and node.op == "call_function"
|
||||
and node.target is target
|
||||
)
|
||||
|
||||
|
||||
def is_method(node: object, name: str) -> TypeIs[fx.Node]:
|
||||
"""Is node `.<name>()`."""
|
||||
return (
|
||||
isinstance(node, fx.Node) and node.op == "call_method" and node.target == name
|
||||
)
|
||||
|
||||
|
||||
def is_op(node: object, name: str) -> TypeIs[fx.Node]:
|
||||
"""Is node `<mod>.<name>()` for torch, F, operator, or Tensor."""
|
||||
return any(
|
||||
is_fn(node, getattr(module, name, None)) for module in (torch, F, operator)
|
||||
) or (hasattr(torch.Tensor, name) and is_method(node, name))
|
||||
@@ -1,64 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Layer provider resolution for the Transformers modeling backend.
|
||||
|
||||
When ``VLLM_USE_HW_AGNOSTIC`` is set, layer symbols are imported from
|
||||
``vllm.model_executor.hw_agnostic.layers.<module>``, falling back to
|
||||
``vllm.model_executor.layers.<module>`` for anything not yet ported. The
|
||||
resolved source of every symbol is logged so it is clear which layers run
|
||||
hw-agnostic and which fell back to vLLM.
|
||||
"""
|
||||
|
||||
import importlib
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_HW_PKG = "vllm.model_executor.hw_agnostic.layers"
|
||||
_VLLM_PKG = "vllm.model_executor.layers"
|
||||
|
||||
|
||||
def _resolve(module: str, name: str):
|
||||
"""Return `name` from the hw-agnostic `module` when enabled and available,
|
||||
else from vLLM. Logs which source was used."""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
obj = getattr(importlib.import_module(f"{_HW_PKG}.{module}"), name)
|
||||
logger.info("Using hw-agnostic layer: %s", name)
|
||||
return obj
|
||||
except (ImportError, AttributeError):
|
||||
logger.warning(
|
||||
"hw-agnostic layer %s is not available; falling back to default", name
|
||||
)
|
||||
return getattr(importlib.import_module(f"{_VLLM_PKG}.{module}"), name)
|
||||
|
||||
|
||||
RMSNorm = _resolve("layernorm", "RMSNorm")
|
||||
GemmaRMSNorm = _resolve("layernorm", "GemmaRMSNorm")
|
||||
|
||||
|
||||
def get_act_and_mul_fn(act_fn_name: str):
|
||||
"""Fused activation-and-mul op for `act_fn_name`, preferring hw-agnostic.
|
||||
|
||||
Resolved per call because the op is name-parameterized: an activation with
|
||||
no hw-agnostic equivalent falls back to vLLM individually.
|
||||
"""
|
||||
if envs.VLLM_USE_HW_AGNOSTIC:
|
||||
try:
|
||||
from vllm.model_executor.hw_agnostic.layers.activation import (
|
||||
get_act_and_mul_fn as hw_fn,
|
||||
)
|
||||
|
||||
fn = hw_fn(act_fn_name)
|
||||
logger.info_once("Using hw-agnostic activation: %s", act_fn_name)
|
||||
return fn
|
||||
except (ImportError, KeyError):
|
||||
logger.warning_once(
|
||||
"hw-agnostic activation %s is not available; falling back to vLLM",
|
||||
act_fn_name,
|
||||
)
|
||||
from vllm.model_executor.layers.activation import get_act_and_mul_fn as vllm_fn
|
||||
|
||||
return vllm_fn(act_fn_name)
|
||||
@@ -1,80 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for legacy models."""
|
||||
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
import torch
|
||||
|
||||
from vllm.model_executor.models.utils import WeightsMapper
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .base import Base
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
|
||||
class LegacyMixin(Base):
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
self.hf_to_vllm_mapper |= WeightsMapper(
|
||||
# Drop unsupported/unwanted output embeddings layers.
|
||||
orig_to_new_prefix={
|
||||
"model.lm_head.": None,
|
||||
"model.predictions.": None,
|
||||
"model.qa_outputs.": None,
|
||||
"model.embeddings_project.": None,
|
||||
"model.discriminator_predictions.": None,
|
||||
},
|
||||
orig_to_new_substr={
|
||||
# Some encoder models have the position_ids buffer in the checkpoint.
|
||||
# vLLM always passes position_ids as an argument, so drop the buffer.
|
||||
"position_ids": None,
|
||||
# Some encoder models have the bias of the final classifier layer in
|
||||
# the checkpoint. vLLM does not use this bias, so drop it.
|
||||
"score.bias": None,
|
||||
},
|
||||
)
|
||||
|
||||
# roberta-like models an extra padding in positions.
|
||||
# FIXME(Isotr0py): This is quite hacky for roberta edge case,
|
||||
# we should find a better way to handle this.
|
||||
self.is_roberta = "roberta" in self.text_config.model_type
|
||||
self.padding_idx = self.text_config.pad_token_id
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors: IntermediateTensors | None = None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | IntermediateTensors:
|
||||
if self.is_roberta:
|
||||
# RoBERTa positions start at padding_idx + 1.
|
||||
# Non-in-place add to avoid mutating the persistent GPU buffer --
|
||||
# in-place += would accumulate on CUDA graph padding slots.
|
||||
positions = positions + self.padding_idx + 1
|
||||
return super().forward(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
inputs_embeds=inputs_embeds,
|
||||
**kwargs,
|
||||
)
|
||||
@@ -1,454 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend mixin for Mixture of Experts (MoE) models."""
|
||||
|
||||
from dataclasses import dataclass
|
||||
from functools import partial
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
from vllm._aiter_ops import rocm_aiter_ops
|
||||
from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_dp_group, get_ep_group
|
||||
from vllm.forward_context import ForwardContext, get_forward_context
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.custom_op import PluggableLayer
|
||||
from vllm.model_executor.layers.fused_moe import (
|
||||
FusedMoEFactory,
|
||||
MoERunner,
|
||||
RoutedExperts,
|
||||
)
|
||||
from vllm.model_executor.models.interfaces import MixtureOfExperts
|
||||
from vllm.model_executor.models.transformers.fuser import get_fuser
|
||||
from vllm.model_executor.models.transformers.fusers.glu import GLUFuser
|
||||
from vllm.model_executor.models.transformers.fusers.moe import MoEBlockFuser
|
||||
from vllm.model_executor.models.utils import extract_layer_index, maybe_prefix
|
||||
from vllm.transformers_utils.config import CONFIG_FIELD_ALIASES
|
||||
from vllm.utils.torch_utils import STR_DTYPE_TO_TORCH_DTYPE, direct_register_custom_op
|
||||
|
||||
from .base import Base
|
||||
from .utils import log_replacement, maybe_per_layer
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TransformersMoEState:
|
||||
topk_ids: torch.Tensor | None = None
|
||||
is_sequence_parallel: bool = False
|
||||
|
||||
|
||||
# --8<-- [start:transformers_fused_moe]
|
||||
@PluggableLayer.register("transformers_fused_moe")
|
||||
class TransformersMoERunner(MoERunner):
|
||||
"""Custom MoERunner for the Transformers modeling backend."""
|
||||
|
||||
# --8<-- [end:transformers_fused_moe]
|
||||
def __init__(self, *args, moe_state: TransformersMoEState, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
self._moe_state = moe_state
|
||||
self._moe_state.is_sequence_parallel = self.moe_config.is_sequence_parallel
|
||||
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
**kwargs: Any,
|
||||
) -> torch.Tensor:
|
||||
"""In Transformers `experts.forward` will have this signature, which is why
|
||||
it deliberately does not match `MoERunner.forward`.
|
||||
|
||||
We discard any extra kwargs because we cannot use them here."""
|
||||
# Note: we need to forward through a custom op so the topk_ids
|
||||
# can be transferred without interfering with cudagraphs.
|
||||
return torch.ops.vllm.transformers_moe_forward(
|
||||
hidden_states,
|
||||
topk_ids.to(torch.int32),
|
||||
topk_weights.to(torch.float32),
|
||||
self.layer_name,
|
||||
)
|
||||
|
||||
def _forward_super(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
return super().forward(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def _transformers_moe_forward(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
"""Store the `topk_ids` in the layer and call the actual forward."""
|
||||
forward_context: ForwardContext = get_forward_context()
|
||||
self = forward_context.no_compile_layers[layer_name]
|
||||
self._moe_state.topk_ids = topk_ids
|
||||
return self._forward_super(hidden_states, topk_weights)
|
||||
|
||||
|
||||
def _transformers_moe_forward_fake(
|
||||
hidden_states: torch.Tensor,
|
||||
topk_ids: torch.Tensor,
|
||||
topk_weights: torch.Tensor,
|
||||
layer_name: str,
|
||||
) -> torch.Tensor:
|
||||
return torch.empty_like(hidden_states)
|
||||
|
||||
|
||||
direct_register_custom_op(
|
||||
op_name="transformers_moe_forward",
|
||||
op_func=_transformers_moe_forward,
|
||||
mutates_args=["hidden_states"],
|
||||
fake_impl=_transformers_moe_forward_fake,
|
||||
tags=(torch.Tag.needs_fixed_stride_order,),
|
||||
)
|
||||
|
||||
|
||||
def _hf_topk_router_precomputed(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
moe_state: TransformersMoEState,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Return `(topk_weights, topk_ids)` the HF block forward already computed:
|
||||
weights arrive as `gating_output`, ids via `moe_state`."""
|
||||
topk_weights = gating_output
|
||||
topk_ids = moe_state.topk_ids
|
||||
assert topk_ids is not None
|
||||
# Handle all gather in expert parallel
|
||||
if topk_ids.size(0) != hidden_states.size(0):
|
||||
dp_metadata = get_forward_context().dp_metadata
|
||||
assert dp_metadata is not None
|
||||
sizes = dp_metadata.get_chunk_sizes_across_dp_rank()
|
||||
assert sizes is not None
|
||||
is_sp = moe_state.is_sequence_parallel
|
||||
group = get_ep_group() if is_sp else get_dp_group()
|
||||
assert sizes[group.rank_in_group] == topk_ids.shape[0]
|
||||
(topk_ids,) = group.all_gatherv([topk_ids], 0, sizes)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
def _hf_topk_router(
|
||||
hidden_states: torch.Tensor,
|
||||
gating_output: torch.Tensor,
|
||||
topk: int,
|
||||
renormalize: bool,
|
||||
gate: nn.Module,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Route with a Transformers gate returning `(_, topk_weights, topk_ids)`."""
|
||||
_, topk_weights, topk_ids = gate(hidden_states)
|
||||
return topk_weights, topk_ids
|
||||
|
||||
|
||||
class TransformersRoutedExperts(RoutedExperts):
|
||||
def get_expert_mapping(
|
||||
self,
|
||||
ckpt_gate_proj_name: str | None = None,
|
||||
ckpt_down_proj_name: str | None = None,
|
||||
ckpt_up_proj_name: str | None = None,
|
||||
include_fused: bool = False,
|
||||
) -> list[tuple[str, str, int, str]]:
|
||||
common_map = super().get_expert_mapping(
|
||||
ckpt_gate_proj_name or "gate_proj",
|
||||
ckpt_down_proj_name or "down_proj",
|
||||
ckpt_up_proj_name or "up_proj",
|
||||
include_fused,
|
||||
)
|
||||
mixtral_map = super().get_expert_mapping("w1", "w2", "w3", include_fused)
|
||||
if not include_fused:
|
||||
return common_map + mixtral_map
|
||||
common_fused, common_unfused = common_map[:3], common_map[3:]
|
||||
mixtral_fused, mixtral_unfused = mixtral_map[:3], mixtral_map[3:]
|
||||
return common_fused + mixtral_fused + common_unfused + mixtral_unfused
|
||||
|
||||
|
||||
class MoEMixin(MixtureOfExperts, Base):
|
||||
mlp_layers: list[nn.Module]
|
||||
"""MoE blocks whose experts were replaced, for the `MixtureOfExperts` methods."""
|
||||
|
||||
# Redeclared because `recursive_replace` assigns it below the first read
|
||||
num_local_physical_experts: int
|
||||
|
||||
def __init__(self, *, vllm_config: "VllmConfig", prefix: str = ""):
|
||||
# Skip MixtureOfExperts.__init__ and call the next class in MRO
|
||||
super(MixtureOfExperts, self).__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
num_local_physical_experts: int,
|
||||
):
|
||||
assert self.num_local_physical_experts == num_local_physical_experts
|
||||
self.num_physical_experts = num_physical_experts
|
||||
self.num_local_physical_experts = num_local_physical_experts
|
||||
self.num_redundant_experts = num_physical_experts - self.num_logical_experts
|
||||
for moe_block in self.mlp_layers:
|
||||
moe_block.n_local_physical_experts = num_local_physical_experts
|
||||
moe_block.n_physical_experts = num_physical_experts
|
||||
moe_block.n_redundant_experts = self.num_redundant_experts
|
||||
moe_block.experts.update_expert_map()
|
||||
|
||||
def recursive_replace(self):
|
||||
"""Initialize the MoE layers."""
|
||||
experts_name = "experts"
|
||||
text_config = self.text_config
|
||||
|
||||
# Positional arguments
|
||||
num_experts_names = CONFIG_FIELD_ALIASES["num_experts"]
|
||||
num_experts = getattr_iter(text_config, num_experts_names, None)
|
||||
top_k_names = CONFIG_FIELD_ALIASES["num_experts_per_token"]
|
||||
top_k = getattr_iter(text_config, top_k_names, None)
|
||||
assert top_k is not None
|
||||
hidden_size = text_config.hidden_size
|
||||
intermediate_size_names = CONFIG_FIELD_ALIASES["moe_intermediate_size"]
|
||||
intermediate_size = getattr_iter(text_config, intermediate_size_names, None)
|
||||
assert intermediate_size is not None
|
||||
|
||||
num_shared_experts_names = CONFIG_FIELD_ALIASES["num_shared_experts"]
|
||||
num_shared_experts = getattr_iter(text_config, num_shared_experts_names, 0)
|
||||
|
||||
# Common kwargs
|
||||
norm_topk_prob = getattr(text_config, "norm_topk_prob", None)
|
||||
|
||||
# Routed scaling factor kwargs
|
||||
routed_scaling_factor = getattr(text_config, "routed_scaling_factor", 1.0)
|
||||
# aiter applies routed_scaling_factor internally
|
||||
apply_routed_scale_to_output = not rocm_aiter_ops.is_fused_moe_enabled()
|
||||
routed_scaling_factor_kwargs = dict(
|
||||
routed_scaling_factor=routed_scaling_factor,
|
||||
apply_routed_scale_to_output=apply_routed_scale_to_output,
|
||||
)
|
||||
|
||||
# Dtype the router computes in, if it is not the activation dtype.
|
||||
config_router_dtype = getattr(text_config, "moe_router_dtype", None)
|
||||
if config_router_dtype is not None:
|
||||
config_router_dtype = STR_DTYPE_TO_TORCH_DTYPE.get(config_router_dtype)
|
||||
|
||||
# Grouped topk routing kwargs
|
||||
num_expert_group = getattr(text_config, "n_group", None)
|
||||
topk_group = getattr(text_config, "topk_group", None)
|
||||
use_grouped_topk = num_expert_group is not None and topk_group is not None
|
||||
grouped_topk_routing_kwargs = dict(
|
||||
use_grouped_topk=use_grouped_topk,
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
)
|
||||
|
||||
# MoE activation function
|
||||
activation = "silu"
|
||||
wrapped_arch = self.config.architectures[0].lower()
|
||||
if "gptoss" in wrapped_arch:
|
||||
activation = "swigluoai"
|
||||
|
||||
# Expert parallel load balancing kwargs
|
||||
enable_eplb = self.parallel_config.enable_eplb
|
||||
num_redundant_experts = self.parallel_config.eplb_config.num_redundant_experts
|
||||
|
||||
# MixtureOfExperts mixin settings
|
||||
ep_size = get_ep_group().world_size
|
||||
|
||||
self.mlp_layers = []
|
||||
self.moe_layers = []
|
||||
self.num_expert_groups = 1 if num_expert_group is None else num_expert_group
|
||||
self.num_logical_experts = num_experts
|
||||
self.num_physical_experts = num_experts + num_redundant_experts
|
||||
self.num_local_physical_experts = self.num_physical_experts // ep_size
|
||||
self.num_routed_experts = num_experts
|
||||
self.num_shared_experts = num_shared_experts
|
||||
self.num_redundant_experts = num_redundant_experts
|
||||
|
||||
# Down projections of shared experts consumed by FusedMoE
|
||||
shared_down_projs: list[tuple[nn.Module, str]] = []
|
||||
|
||||
# Recursively fuse MoE layers
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
# Naive implementations will have experts as ModuleList
|
||||
is_modulelist = isinstance(child_module, nn.ModuleList)
|
||||
# Packed implementations will have experts as 3D tensors of shapes like:
|
||||
# gate_up_proj = (num_experts, 2 * intermediate_size, hidden_size)
|
||||
# down_proj = (num_experts, intermediate_size, hidden_size)
|
||||
params = list(child_module.parameters())
|
||||
is_3d = len(params) > 0 and all(p.ndim == 3 for p in params)
|
||||
if child_name == experts_name and (is_modulelist or is_3d):
|
||||
# Alias for readability
|
||||
moe_block = module
|
||||
experts = child_module
|
||||
# Class of the fused block (parent of gate/experts/shared)
|
||||
moe_block_cls = type(moe_block).__name__
|
||||
experts_cls = type(experts).__name__
|
||||
# Do the experts have biases
|
||||
has_bias = False
|
||||
for experts_param_name, _ in experts.named_parameters():
|
||||
if "bias" in experts_param_name:
|
||||
has_bias = True
|
||||
break
|
||||
# Whether the expert weights are stored in transposed format.
|
||||
# Comes from Transformers `use_experts_implementation` decorator.
|
||||
is_transposed = getattr(experts, "is_transposed", False)
|
||||
# If the config does not specify num_shared_experts, but
|
||||
# the model has shared experts, we assume there is one.
|
||||
if self.num_shared_experts == 0:
|
||||
for moe_block_param_name, _ in moe_block.named_parameters():
|
||||
if "shared_expert" in moe_block_param_name:
|
||||
self.num_shared_experts = 1
|
||||
break
|
||||
|
||||
# Only pay for the layer index if something is per-layer;
|
||||
# `extract_layer_index` raises for unnumbered module paths.
|
||||
per_layer = (top_k, intermediate_size, norm_topk_prob)
|
||||
layer_idx = (
|
||||
extract_layer_index(qual_name)
|
||||
if any(isinstance(v, list) for v in per_layer)
|
||||
else 0
|
||||
)
|
||||
layer_top_k = maybe_per_layer(top_k, layer_idx)
|
||||
layer_renormalize = maybe_per_layer(norm_topk_prob, layer_idx)
|
||||
if layer_renormalize is None:
|
||||
layer_renormalize = layer_top_k > 1
|
||||
|
||||
kwargs: dict[str, Any] = dict(
|
||||
num_experts=num_experts,
|
||||
top_k=layer_top_k,
|
||||
hidden_size=hidden_size,
|
||||
intermediate_size=maybe_per_layer(intermediate_size, layer_idx),
|
||||
renormalize=layer_renormalize,
|
||||
quant_config=self.quant_config,
|
||||
prefix=qual_name,
|
||||
activation=activation,
|
||||
enable_eplb=enable_eplb,
|
||||
num_redundant_experts=num_redundant_experts,
|
||||
has_bias=has_bias,
|
||||
is_fused_checkpoint_transposed=is_transposed,
|
||||
routed_experts_cls=TransformersRoutedExperts,
|
||||
)
|
||||
# _maybe_apply_routed_scale_to_output edge case. Transformers
|
||||
# decoder layers do not compensate for dividing by scaling factor.
|
||||
divides_shared_output = (
|
||||
routed_scaling_factor != 1.0
|
||||
and apply_routed_scale_to_output
|
||||
and self.model_config.dtype == torch.float16
|
||||
)
|
||||
fuser = MoEBlockFuser.match(
|
||||
moe_block, experts_name, divides_shared_output
|
||||
)
|
||||
if fuser is not None:
|
||||
# MoE block forward is fully replaced.
|
||||
# gate/router and shared expert (if any) runs in MoERunner.
|
||||
shared_experts = fuser.shared_experts(moe_block, prefix)
|
||||
# Store shared experts for later down projection adjustment
|
||||
if shared_experts is not None:
|
||||
hf_shared = shared_experts.shared_experts
|
||||
glu_fuser = get_fuser(hf_shared, GLUFuser)
|
||||
if glu_fuser is not None:
|
||||
down_name = glu_fuser.down_name
|
||||
if down_name is not None:
|
||||
shared_down_projs.append((hf_shared, down_name))
|
||||
if scoring_func := fuser.scoring_func:
|
||||
# Internal routing kwargs.
|
||||
# Prefer config, otherwise read it from fuser.
|
||||
router_dtype = config_router_dtype or fuser.router_dtype
|
||||
gate = fuser.gate(moe_block, prefix, router_dtype)
|
||||
kwargs |= dict(
|
||||
scoring_func=scoring_func,
|
||||
renormalize=(
|
||||
layer_renormalize
|
||||
if fuser.renormalize is None
|
||||
else fuser.renormalize
|
||||
),
|
||||
gate=gate,
|
||||
)
|
||||
if router_dtype is not None:
|
||||
kwargs["router_logits_dtype"] = router_dtype
|
||||
if use_grouped_topk:
|
||||
kwargs |= grouped_topk_routing_kwargs
|
||||
if routed_scaling_factor != 1.0:
|
||||
kwargs |= routed_scaling_factor_kwargs
|
||||
bias = getattr(gate, "e_score_correction_bias", None)
|
||||
if bias is not None:
|
||||
kwargs["e_score_correction_bias"] = bias
|
||||
else:
|
||||
# External routing kwargs
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
_hf_topk_router,
|
||||
gate=getattr(moe_block, fuser.gate_name),
|
||||
),
|
||||
)
|
||||
|
||||
kwargs |= dict(
|
||||
is_sequence_parallel=(
|
||||
self.parallel_config.use_sequence_parallel_moe
|
||||
),
|
||||
shared_experts=shared_experts,
|
||||
)
|
||||
fuser.rewrite_forward(moe_block)
|
||||
logger.info_once(
|
||||
"Fused: gate + experts%s (%s) -> MoERunner (%s routing)",
|
||||
" + shared experts" if fuser.shared_name else "",
|
||||
moe_block_cls,
|
||||
"external" if fuser.scoring_func is None else "internal",
|
||||
)
|
||||
else:
|
||||
# MoE block forward is unmodified.
|
||||
# gate/router and shared expert (if any) runs in Transformers.
|
||||
# We then smuggle the topk_ids in using a custom op.
|
||||
moe_state = TransformersMoEState()
|
||||
|
||||
kwargs |= dict(
|
||||
num_expert_group=num_expert_group,
|
||||
topk_group=topk_group,
|
||||
custom_routing_function=partial(
|
||||
_hf_topk_router_precomputed, moe_state=moe_state
|
||||
),
|
||||
runner_cls=TransformersMoERunner,
|
||||
runner_args={"moe_state": moe_state},
|
||||
)
|
||||
logger.info_once(
|
||||
"Fused: experts (%s) -> MoERunner (external routing)",
|
||||
experts_cls,
|
||||
)
|
||||
fused_experts = FusedMoEFactory(**kwargs)
|
||||
moe_block.experts = fused_experts
|
||||
log_replacement(qual_name, experts, fused_experts)
|
||||
# Update MixtureOfExperts mixin state
|
||||
self.mlp_layers.append(moe_block)
|
||||
self.moe_layers.append(fused_experts)
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
self.num_moe_layers = len(self.moe_layers)
|
||||
# Continue with the replacement of layers in Base
|
||||
super().recursive_replace()
|
||||
# GLUFuser likely fused shared_experts. The down projection after the GLU
|
||||
# normally immediately reduces but we want FusedMoE to handle the reduction.
|
||||
for hf_shared, down_name in shared_down_projs:
|
||||
hf_shared.get_submodule(down_name).reduce_results = False
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,407 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
# Copyright 2024 The vLLM team.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""Transformers modeling backend utilities."""
|
||||
|
||||
from collections.abc import Callable, Iterator
|
||||
from contextlib import contextmanager
|
||||
from itertools import chain
|
||||
from operator import attrgetter
|
||||
from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Literal, TypeAlias, TypeVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.conv import Conv2dLayer, Conv3dLayer
|
||||
from vllm.model_executor.layers.layernorm import LayerNorm
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.models.utils import maybe_prefix
|
||||
from vllm.transformers_utils.config import iter_rope_parameters
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
# Copied from `accelerate`
|
||||
@contextmanager
|
||||
def init_on_device_without_buffers(device: torch.device):
|
||||
"""A context manager under which models are initialized with all
|
||||
parameters on the specified device. However buffers are not
|
||||
initialized on specified device.
|
||||
|
||||
Args:
|
||||
device (`torch.device`):
|
||||
Device to initialize all parameters on.
|
||||
|
||||
"""
|
||||
old_register_parameter = nn.Module.register_parameter
|
||||
|
||||
def register_empty_parameter(module, name, param):
|
||||
old_register_parameter(module, name, param)
|
||||
if param is not None:
|
||||
param_cls = type(module._parameters[name])
|
||||
kwargs = module._parameters[name].__dict__
|
||||
kwargs["requires_grad"] = param.requires_grad
|
||||
module._parameters[name] = param_cls(
|
||||
module._parameters[name].to(device), **kwargs
|
||||
)
|
||||
|
||||
tensor_constructors_to_patch: dict[str, Callable] = {}
|
||||
|
||||
def patch_tensor_constructor(fn):
|
||||
def wrapper(*args, **kwargs):
|
||||
kwargs["device"] = device
|
||||
return fn(*args, **kwargs)
|
||||
|
||||
return wrapper
|
||||
|
||||
try:
|
||||
nn.Module.register_parameter = register_empty_parameter
|
||||
for torch_function_name in tensor_constructors_to_patch:
|
||||
setattr(
|
||||
torch,
|
||||
torch_function_name,
|
||||
patch_tensor_constructor(getattr(torch, torch_function_name)),
|
||||
)
|
||||
yield
|
||||
finally:
|
||||
nn.Module.register_parameter = old_register_parameter
|
||||
for (
|
||||
torch_function_name,
|
||||
old_torch_function,
|
||||
) in tensor_constructors_to_patch.items():
|
||||
setattr(torch, torch_function_name, old_torch_function)
|
||||
|
||||
|
||||
Style = Literal[
|
||||
"colwise",
|
||||
"rowwise",
|
||||
"replicate",
|
||||
"colwise_gather_output",
|
||||
"rowwise_split_input",
|
||||
]
|
||||
|
||||
|
||||
def replace_linear_class(
|
||||
linear: nn.Linear,
|
||||
style: Style = "replicate",
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> ColumnParallelLinear | RowParallelLinear | ReplicatedLinear:
|
||||
"""Replace nn.Linear with one of vLLM's tensor parallel linear classes.
|
||||
|
||||
Args:
|
||||
linear: `nn.Linear` to be replaced.
|
||||
style: Tensor parallel style of the new linear, e.g. "colwise".
|
||||
quant_config: Quantization config for the new linear.
|
||||
prefix: Module prefix of the new linear, used for quantization lookup.
|
||||
|
||||
Returns:
|
||||
The new linear.
|
||||
|
||||
"""
|
||||
if not isinstance(style, str):
|
||||
raise ValueError(f"Unsupported parallel style type {type(style)}, expected str")
|
||||
|
||||
vllm_linear_cls, vllm_linear_kwargs = {
|
||||
"colwise": (ColumnParallelLinear, {}),
|
||||
"rowwise": (RowParallelLinear, {}),
|
||||
"replicate": (ReplicatedLinear, {}),
|
||||
"colwise_gather_output": (ColumnParallelLinear, {"gather_output": True}),
|
||||
"rowwise_split_input": (RowParallelLinear, {"input_is_parallel": False}),
|
||||
}.get(style, (ReplicatedLinear, {}))
|
||||
|
||||
return vllm_linear_cls(
|
||||
input_size=linear.in_features,
|
||||
output_size=linear.out_features,
|
||||
bias=linear.bias is not None,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
return_bias=False,
|
||||
**vllm_linear_kwargs,
|
||||
)
|
||||
|
||||
|
||||
TorchConv: TypeAlias = nn.Conv2d | nn.Conv3d
|
||||
VllmConv: TypeAlias = Conv2dLayer | Conv3dLayer
|
||||
|
||||
|
||||
def replace_conv_class(conv: TorchConv) -> VllmConv | TorchConv:
|
||||
"""Replace a Transformers Conv2d/Conv3d with vLLM's Conv2d/Conv3d.
|
||||
|
||||
Args:
|
||||
conv: `nn.Conv2d` or `nn.Conv3d` to be replaced.
|
||||
|
||||
Returns:
|
||||
The new `Conv2dLayer` or `Conv3dLayer`. If the conv module is not supported,
|
||||
returns the original conv module.
|
||||
|
||||
"""
|
||||
# vLLM does not handle non-zero padding modes
|
||||
if conv.padding_mode != "zeros":
|
||||
return conv
|
||||
|
||||
vllm_conv_cls = {
|
||||
nn.Conv2d: Conv2dLayer,
|
||||
nn.Conv3d: Conv3dLayer,
|
||||
}.get(type(conv))
|
||||
|
||||
if vllm_conv_cls is None:
|
||||
return conv
|
||||
|
||||
return vllm_conv_cls(
|
||||
in_channels=conv.in_channels,
|
||||
out_channels=conv.out_channels,
|
||||
kernel_size=conv.kernel_size,
|
||||
stride=conv.stride,
|
||||
padding=conv.padding,
|
||||
dilation=conv.dilation,
|
||||
groups=conv.groups,
|
||||
bias=conv.bias is not None,
|
||||
padding_mode=conv.padding_mode,
|
||||
params_dtype=conv.weight.dtype,
|
||||
)
|
||||
|
||||
|
||||
def replace_layernorm_class(layernorm: nn.LayerNorm) -> nn.Module:
|
||||
"""Replace a standard (mean-centered) `nn.LayerNorm` with vLLM's
|
||||
`LayerNorm`.
|
||||
|
||||
Args:
|
||||
layernorm: `nn.LayerNorm` to be replaced.
|
||||
|
||||
Returns:
|
||||
The new `LayerNorm`. If the layernorm is not supported (a subclass with
|
||||
its own behavior, multi-dim `normalized_shape`, or not both
|
||||
elementwise-affine and biased), returns the original module unchanged.
|
||||
|
||||
"""
|
||||
if (
|
||||
type(layernorm) is not nn.LayerNorm
|
||||
or len(layernorm.normalized_shape) != 1
|
||||
or not layernorm.elementwise_affine
|
||||
or layernorm.bias is None
|
||||
):
|
||||
return layernorm
|
||||
|
||||
return LayerNorm(
|
||||
layernorm.normalized_shape[0],
|
||||
eps=layernorm.eps,
|
||||
dtype=layernorm.weight.dtype,
|
||||
)
|
||||
|
||||
|
||||
def attrsetter(attr: str) -> Callable[[object, object], None]:
|
||||
"""Set a possibly nested attribute, like the inverse of attrgetter."""
|
||||
parent, _, name = attr.rpartition(".")
|
||||
|
||||
def setter(obj: object, value: object):
|
||||
attr_parent = attrgetter(parent)(obj) if parent else obj
|
||||
setattr(attr_parent, name, value)
|
||||
|
||||
return setter
|
||||
|
||||
|
||||
class _UninitializedEmbedding(nn.Embedding):
|
||||
"""Make `__init__` inert, so that `VocabParallelEmbedding.__init__`'
|
||||
call to `super().__init__` does not invoke `nn.Embedding.__init__`."""
|
||||
|
||||
def __init__(self):
|
||||
pass
|
||||
|
||||
|
||||
class _VocabParallelEmbeddingBase(VocabParallelEmbedding, _UninitializedEmbedding):
|
||||
"""Orders `VocabParallelEmbedding` ahead of `nn.Embedding` in the MRO, so that
|
||||
`super().forward(...)` in an `nn.Embedding` subclass reaches vLLM's embedding."""
|
||||
|
||||
|
||||
_rebased_embedding_classes: dict[type[nn.Embedding], type[VocabParallelEmbedding]] = {}
|
||||
|
||||
|
||||
def _rebase_on_vocab_parallel(cls: type[nn.Embedding]) -> type[VocabParallelEmbedding]:
|
||||
"""Subclass `cls` so that `VocabParallelEmbedding` supersedes its `nn.Embedding`.
|
||||
|
||||
Args:
|
||||
cls: The `nn.Embedding` subclass to rebase. Cached, so a given `cls` always
|
||||
maps to the same class.
|
||||
|
||||
Returns:
|
||||
The new class, to assign to `__class__` of an instance of `cls`.
|
||||
|
||||
"""
|
||||
if cls not in _rebased_embedding_classes:
|
||||
rebased_cls = type(cls.__name__, (cls, _VocabParallelEmbeddingBase), {})
|
||||
_rebased_embedding_classes[cls] = rebased_cls
|
||||
return _rebased_embedding_classes[cls]
|
||||
|
||||
|
||||
def replace_embedding_class(
|
||||
embedding: nn.Embedding,
|
||||
quant_config: "QuantizationConfig | None" = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> nn.Module:
|
||||
"""Replace `nn.Embedding` with `VocabParallelEmbedding`.
|
||||
|
||||
Args:
|
||||
embedding: The `nn.Embedding` holding a vocab table.
|
||||
quant_config: Quantization config for the new embedding.
|
||||
prefix: Qualname of `embedding`, used to look up its quantization method.
|
||||
|
||||
Returns:
|
||||
The module to install in `embedding`'s place.
|
||||
|
||||
"""
|
||||
# Tied tables are reached more than once, so don't rebase an already new embedding
|
||||
if isinstance(embedding, VocabParallelEmbedding):
|
||||
return embedding
|
||||
|
||||
kwargs = dict(
|
||||
num_embeddings=embedding.num_embeddings,
|
||||
embedding_dim=embedding.embedding_dim,
|
||||
params_dtype=embedding.weight.dtype,
|
||||
quant_config=quant_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
# If `embedding` is a bare `nn.Embedding`, simple replace
|
||||
if type(embedding) is nn.Embedding:
|
||||
return VocabParallelEmbedding(**kwargs)
|
||||
|
||||
# Otherwise `embedding` inherits `nn.Embedding`, rebase it in place
|
||||
embedding.__class__ = _rebase_on_vocab_parallel(type(embedding))
|
||||
VocabParallelEmbedding.__init__(embedding, **kwargs)
|
||||
return embedding
|
||||
|
||||
|
||||
def recursive_replace_linear(
|
||||
model: nn.Module,
|
||||
quant_config: "QuantizationConfig | None",
|
||||
prefix: str = "",
|
||||
):
|
||||
"""Recursively replace linear modules in the model as needed."""
|
||||
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
new_module = child_module
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
# Replace modules as needed
|
||||
if isinstance(child_module, nn.Linear):
|
||||
new_module = replace_linear_class(
|
||||
child_module,
|
||||
"replicate",
|
||||
quant_config,
|
||||
prefix=qual_name,
|
||||
)
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
if new_module is not child_module:
|
||||
setattr(module, child_name, new_module)
|
||||
|
||||
_recursive_replace(model, prefix=prefix)
|
||||
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def maybe_per_layer(value: T | list[T], layer_idx: int) -> T:
|
||||
"""Pick `layer_idx`'s entry from a config field that may be sized per layer."""
|
||||
return value[layer_idx] if isinstance(value, list) else value
|
||||
|
||||
|
||||
def named_state(module: nn.Module) -> Iterator[tuple[str, torch.Tensor]]:
|
||||
"""`module`'s own state (i.e. named parameters and buffers)."""
|
||||
return chain(module.named_parameters(), module.named_buffers())
|
||||
|
||||
|
||||
def log_replacement(name: str, old_module: nn.Module, new_module: nn.Module):
|
||||
logger.debug("%s: %s -> %s", name, old_module, new_module)
|
||||
|
||||
|
||||
def get_feature_request_tip(
|
||||
model: str,
|
||||
trust_remote_code: bool,
|
||||
) -> str:
|
||||
hf_url = f"a discussion at https://huggingface.co/{model}/discussions/new"
|
||||
gh_url = "an issue at https://github.com/huggingface/transformers/issues/new/choose"
|
||||
url = hf_url if trust_remote_code else gh_url
|
||||
prefix = f"Please open {url} to request support for this feature. "
|
||||
if Path(model).exists():
|
||||
prefix = ""
|
||||
doc_url = "https://docs.vllm.ai/en/latest/models/supported_models.html#writing-custom-models"
|
||||
tip = f"See {doc_url} for instructions on how to add support yourself."
|
||||
return f"{prefix}{tip}"
|
||||
|
||||
|
||||
def dynamic_rope_update_is_noop(vllm_config: "VllmConfig") -> bool:
|
||||
"""Whether `max_model_len` keeps Transformers' dynamic rope from ever rescaling.
|
||||
|
||||
`@dynamic_rope_update` only recomputes `inv_freq` once positions grow past
|
||||
`max_position_embeddings`, which vLLM never feeds beyond `max_model_len`.
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
max_position_embeddings = getattr(text_config, "max_position_embeddings", None)
|
||||
if max_position_embeddings is None:
|
||||
return False
|
||||
return vllm_config.model_config.max_model_len <= max_position_embeddings
|
||||
|
||||
|
||||
def can_enable_torch_compile(vllm_config: "VllmConfig") -> bool:
|
||||
"""Callable to be passed to `@support_torch_compile`'s `enable_if` argument.
|
||||
|
||||
Defaults to `True` but is disabled in the following situations:
|
||||
|
||||
- The model uses dynamic rope scaling that can still rescale within
|
||||
`max_model_len` (otherwise `disable_noop_dynamic_rope` removes it).
|
||||
"""
|
||||
text_config = vllm_config.model_config.hf_config.get_text_config()
|
||||
rope_types = {rp["rope_type"] for rp in iter_rope_parameters(text_config)}
|
||||
if "dynamic" not in rope_types:
|
||||
return True
|
||||
# Layer types using longrope still sync, even once dynamic rope is disabled
|
||||
return "longrope" not in rope_types and dynamic_rope_update_is_noop(vllm_config)
|
||||
|
||||
|
||||
def disable_noop_dynamic_rope(model: nn.Module, vllm_config: "VllmConfig") -> None:
|
||||
"""Skip `@dynamic_rope_update` in rotary modules where it can never rescale.
|
||||
|
||||
The update syncs on `torch.max(position_ids)` every forward, which breaks
|
||||
torch.compile and CUDA graph capture. Relabelling the module's `rope_type` as
|
||||
"default" skips it while keeping the `inv_freq` computed at init.
|
||||
"""
|
||||
if not dynamic_rope_update_is_noop(vllm_config):
|
||||
return
|
||||
for module in model.modules():
|
||||
rope_type = getattr(module, "rope_type", None)
|
||||
if not hasattr(module, "max_seq_len_cached") or rope_type is None:
|
||||
continue
|
||||
if isinstance(rope_type, dict):
|
||||
module.rope_type = {
|
||||
k: "default" if v == "dynamic" else v for k, v in rope_type.items()
|
||||
}
|
||||
elif rope_type == "dynamic":
|
||||
module.rope_type = "default"
|
||||
Reference in New Issue
Block a user