FusedMoE support for the Transformers backend (#22650)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -22,6 +22,8 @@ from typing import Literal, Optional, Union
|
||||
|
||||
import regex as re
|
||||
import torch
|
||||
import transformers
|
||||
from packaging.version import Version
|
||||
from torch import nn
|
||||
from transformers import (AutoModel, BatchFeature, PretrainedConfig,
|
||||
PreTrainedModel)
|
||||
@@ -35,6 +37,7 @@ from vllm.config.utils import getattr_iter
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.utils import get_pp_indices
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear)
|
||||
@@ -121,10 +124,14 @@ def can_enable_torch_compile(vllm_config: VllmConfig) -> bool:
|
||||
return enable
|
||||
|
||||
|
||||
Style = Literal["colwise", "colwise_rep", "rowwise", "rowwise_rep",
|
||||
"replicate"]
|
||||
|
||||
|
||||
def replace_linear_class(
|
||||
linear: nn.Linear,
|
||||
style: Literal["colwise", "rowwise"],
|
||||
quant_config: QuantizationConfig,
|
||||
style: Style = "replicate",
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
*,
|
||||
prefix: str = "",
|
||||
) -> Union[ColumnParallelLinear, RowParallelLinear, ReplicatedLinear]:
|
||||
@@ -132,11 +139,11 @@ def replace_linear_class(
|
||||
Replace nn.Linear with one of vLLM's tensor parallel linear classes.
|
||||
|
||||
Args:
|
||||
linear (nn.Linear): `nn.Linear` to be replaced.
|
||||
style (str): Tensor parallel style of the new linear, e.g. "colwise".
|
||||
quant_config (QuantConfig): Quantization config for the new linear.
|
||||
linear: `nn.Linear` to be replaced.
|
||||
style: Tensor parallel style of the new linear, e.g. "colwise".
|
||||
quant_config: Quantization config for the new linear.
|
||||
Returns:
|
||||
Union[ColumnParallelLinear, RowParallelLinear]: The new linear.
|
||||
The new linear.
|
||||
"""
|
||||
|
||||
if not isinstance(style, str):
|
||||
@@ -166,6 +173,31 @@ def replace_linear_class(
|
||||
)
|
||||
|
||||
|
||||
def replace_rms_norm_class(rms_norm: nn.Module, hidden_size: int) -> RMSNorm:
|
||||
"""Replace a Transformers RMSNorm with vLLM's RMSNorm.
|
||||
|
||||
This method assumes:
|
||||
- Weight is stored as `weight`.
|
||||
- Epsilon is stored as `eps` or `variance_epsilon`.
|
||||
- `with_scale` indicates whether the layer has a weight (Gemma3n only).
|
||||
- `var_hidden_size` is only ever used for Intern vision encoder in vLLM
|
||||
and Transformers doesn't appear to have the same concept.
|
||||
"""
|
||||
kwargs = {
|
||||
"hidden_size": hidden_size,
|
||||
"eps": getattr_iter(rms_norm, ("eps", "variance_epsilon"), 1e-6),
|
||||
"has_weight": getattr(rms_norm, "with_scale", True)
|
||||
}
|
||||
if (weight := getattr(rms_norm, "weight", None)) is not None:
|
||||
# If weight is a Parameter, get its data tensor
|
||||
weight = getattr(weight, "data", weight)
|
||||
kwargs["dtype"] = weight.dtype
|
||||
else:
|
||||
# No weight, fall back to weightless RMSNorm
|
||||
kwargs["has_weight"] = False
|
||||
return RMSNorm(**kwargs)
|
||||
|
||||
|
||||
# Copied from `accelerate`
|
||||
@contextmanager
|
||||
def init_on_device_without_buffers(device: torch.device):
|
||||
@@ -463,9 +495,15 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
self.ignore_unexpected_suffixes: list[str] = []
|
||||
"""Ignore unexpected weights whose qualname ends with these suffixes."""
|
||||
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if self.quant_config and "gptq" in self.quant_config.get_name():
|
||||
self.ignore_unexpected_suffixes.append(".bias")
|
||||
if self.quant_config:
|
||||
quant_method_name = self.quant_config.get_name()
|
||||
# Check for unsupported quantization methods.
|
||||
if quant_method_name == "mxfp4":
|
||||
raise NotImplementedError("Transformers backend does not "
|
||||
"support MXFP4 quantization yet.")
|
||||
# Skip loading extra bias for GPTQ models.
|
||||
if "gptq" in quant_method_name:
|
||||
self.ignore_unexpected_suffixes.append(".bias")
|
||||
|
||||
# Set correct attn and init on "meta" to delay allocating GPU tensors
|
||||
# TODO: @raushan, use the public `model.set_attn_implementation()`
|
||||
@@ -478,8 +516,12 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
trust_remote_code=self.model_config.trust_remote_code,
|
||||
)
|
||||
|
||||
# Remove layers not on this pipeline parallel rank
|
||||
self.pipeline_parallel()
|
||||
self.tensor_parallel()
|
||||
# Substitute remaining layers with vLLM's layers as needed
|
||||
self.recursive_replace()
|
||||
# Create attention instances for KV cache allocation
|
||||
self.attention_instances = self.create_attention_instances()
|
||||
|
||||
# Input embeddings
|
||||
if not isinstance(self.model.get_input_embeddings(), PPMissingLayer):
|
||||
@@ -494,12 +536,10 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
quant_config=self.quant_config,
|
||||
))
|
||||
|
||||
# Attention layers
|
||||
self.attention_instances = self.create_attention_instances()
|
||||
|
||||
# Initialize any parameters that have not had their modules replaced
|
||||
self.init_parameters(self.model)
|
||||
|
||||
# Pipeline parallel intermediate tensors
|
||||
self.make_empty_intermediate_tensors = (
|
||||
make_empty_intermediate_tensors_factory(
|
||||
["hidden_states"], self.text_config.hidden_size))
|
||||
@@ -558,56 +598,53 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
if not self.pp_group.is_last_rank:
|
||||
setattr(self.model, name, PPMissingLayer())
|
||||
|
||||
def tensor_parallel(self):
|
||||
"""
|
||||
Apply the model's tensor parallelization plan.
|
||||
Currently only supports linear layers.
|
||||
"""
|
||||
# Look for tp plans in all of the PreTrainedModels found in self.model
|
||||
is_pretrained_model = lambda m: isinstance(m, PreTrainedModel)
|
||||
supports_tp_plan = lambda m: m.config.base_model_tp_plan is not None
|
||||
pretrained_models = filter(is_pretrained_model, self.model.modules())
|
||||
models_with_tp_plan = filter(supports_tp_plan, pretrained_models)
|
||||
def recursive_replace(self):
|
||||
"""Recursively replace modules in the model as needed.
|
||||
|
||||
if not any(models_with_tp_plan) and self.tp_size > 1:
|
||||
Currently, this replaces:
|
||||
|
||||
- `nn.Linear` with vLLM's tensor parallel linear classes
|
||||
- `*RMSNorm` with vLLM's `RMSNorm`
|
||||
"""
|
||||
tp_plan = self.model.tp_plan
|
||||
|
||||
if not tp_plan and self.tp_size > 1:
|
||||
tip = get_feature_request_tip(self.model_config.model,
|
||||
self.model_config.trust_remote_code)
|
||||
raise ValueError(
|
||||
f"{type(self.model)} does not support tensor parallel. {tip}")
|
||||
|
||||
def _tensor_parallel(module: nn.Module, prefix: str, tp_plan=None):
|
||||
tp_plan = tp_plan or {}
|
||||
# Prefix the patterns because we always start from `self.model`
|
||||
tp_plan = {maybe_prefix("model", k): v for k, v in tp_plan.items()}
|
||||
|
||||
# If the current module is a PreTrainedModel, set the tp_plan for
|
||||
# all of its children
|
||||
if isinstance(module, PreTrainedModel):
|
||||
tp_plan = module.config.base_model_tp_plan or {}
|
||||
tp_plan = {
|
||||
maybe_prefix(prefix, k): v
|
||||
for k, v in tp_plan.items()
|
||||
}
|
||||
|
||||
# Some weight loaders expect linear layers to inherit from vLLM's
|
||||
# LinearBase class, so we set a default style which causes any
|
||||
# unspecified linear layers to be replaced with ReplicatedLinear
|
||||
def _recursive_replace(module: nn.Module, prefix: str):
|
||||
for child_name, child_module in module.named_children():
|
||||
new_module = child_module
|
||||
qual_name = maybe_prefix(prefix, child_name)
|
||||
if isinstance(child_module, nn.Linear):
|
||||
generator = (p for p in tp_plan if re.match(p, qual_name))
|
||||
pattern = next(generator, None)
|
||||
# Some weight loaders expect all linear layers to inherit
|
||||
# LinearBase, so we set a default style which causes any
|
||||
# unspecified layers to be replaced with ReplicatedLinear
|
||||
style = tp_plan.get(pattern, "replicate")
|
||||
new_module = replace_linear_class(child_module,
|
||||
style,
|
||||
self.quant_config,
|
||||
prefix=qual_name)
|
||||
# TODO(hmellor): Enable RMSNorm replacement once we have a way
|
||||
# to choose RMSNorm vs GemmaRMSNorm
|
||||
# elif child_module.__class__.__name__.endswith("RMSNorm"):
|
||||
# new_module = replace_rms_norm_class(
|
||||
# child_module, self.config.hidden_size)
|
||||
else:
|
||||
_recursive_replace(child_module, prefix=qual_name)
|
||||
|
||||
if new_module is not child_module:
|
||||
setattr(module, child_name, new_module)
|
||||
log_replacement(qual_name, child_module, new_module)
|
||||
else:
|
||||
_tensor_parallel(child_module,
|
||||
prefix=qual_name,
|
||||
tp_plan=tp_plan)
|
||||
|
||||
_tensor_parallel(self.model, prefix="model")
|
||||
_recursive_replace(self.model, prefix="model")
|
||||
|
||||
def create_attention_instances(
|
||||
self,
|
||||
@@ -657,15 +694,21 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
self.model: PreTrainedModel = AutoModel.from_config(...)
|
||||
```
|
||||
"""
|
||||
for name, param in module.named_parameters(recurse=False):
|
||||
if param.device == torch.device("meta"):
|
||||
new_param = nn.Parameter(
|
||||
torch.empty_like(param.data,
|
||||
dtype=dtype or self.model_config.dtype,
|
||||
device=self.device_config.device))
|
||||
setattr(module, name, new_param)
|
||||
for child in module.children():
|
||||
self.init_parameters(child, dtype)
|
||||
|
||||
def _init_parameters(module: nn.Module, dtype: Optional[torch.dtype]):
|
||||
for name, param in module.named_parameters(recurse=False):
|
||||
if param.device == torch.device("meta"):
|
||||
new_param = nn.Parameter(
|
||||
torch.empty_like(
|
||||
param.data,
|
||||
dtype=dtype or self.model_config.dtype,
|
||||
device=self.device_config.device,
|
||||
))
|
||||
setattr(module, name, new_param)
|
||||
for child in module.children():
|
||||
_init_parameters(child, dtype)
|
||||
|
||||
_init_parameters(module, dtype)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
@@ -702,8 +745,10 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
|
||||
return hidden_states
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str,
|
||||
torch.Tensor]]) -> set[str]:
|
||||
def load_weights(
|
||||
self,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
) -> set[str]:
|
||||
loader = AutoWeightsLoader(
|
||||
self,
|
||||
skip_prefixes=self.skip_prefixes,
|
||||
@@ -713,6 +758,14 @@ class TransformersBase(nn.Module, SupportsQuant, SupportsLoRA, SupportsPP):
|
||||
)
|
||||
return loader.load_weights(weights, mapper=self.hf_to_vllm_mapper)
|
||||
|
||||
def check_version(self, min_version: str, feature: str):
|
||||
installed = Version(transformers.__version__)
|
||||
required = Version(min_version)
|
||||
if installed < required:
|
||||
raise ImportError(
|
||||
f"Transformers backend requires transformers>={required} "
|
||||
f"for {feature}, but got {installed}")
|
||||
|
||||
|
||||
@support_torch_compile(enable_if=can_enable_torch_compile)
|
||||
class TransformersForCausalLM(TransformersBase):
|
||||
|
||||
Reference in New Issue
Block a user