[Model]: Add transformers backend support (#11330)
# Adds support for `transformers` as a backend Following https://github.com/huggingface/transformers/pull/35235, a bunch of models should already be supported, we are ramping up support for more models. Thanks @Isotr0py for the TP support, and @hmellor for his help as well! This includes: - `trust_remote_code=True` support: any model on the hub, if it implements attention the correct way can be natively supported!! - tensor parallel support --------- Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <41363108+Isotr0py@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com> Co-authored-by: Michael Goin <mgoin64@gmail.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
co-authored by
Isotr0py
Harry Mellor
Isotr0py
Cyrus Leung
Michael Goin
Isotr0py
parent
1298a400e8
commit
a1a2aaadb9
@@ -2,17 +2,22 @@
|
||||
"""Utilities for selecting and loading models."""
|
||||
import contextlib
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Dict, List, Tuple, Type
|
||||
from typing import Dict, List, Optional, Tuple, Type
|
||||
|
||||
import torch
|
||||
import transformers
|
||||
from torch import nn
|
||||
from transformers.dynamic_module_utils import get_class_from_dynamic_module
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.config import ModelConfig, ModelImpl
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.models import ModelRegistry
|
||||
from vllm.model_executor.models.adapters import (as_classification_model,
|
||||
as_embedding_model,
|
||||
as_reward_model)
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@contextlib.contextmanager
|
||||
def set_default_torch_dtype(dtype: torch.dtype):
|
||||
@@ -23,6 +28,50 @@ def set_default_torch_dtype(dtype: torch.dtype):
|
||||
torch.set_default_dtype(old_dtype)
|
||||
|
||||
|
||||
def is_transformers_impl_compatible(
|
||||
arch: str,
|
||||
module: Optional[transformers.PreTrainedModel] = None) -> bool:
|
||||
mod = module or getattr(transformers, arch, None)
|
||||
if mod is None:
|
||||
return False
|
||||
if hasattr(mod, "supports_backend"):
|
||||
return mod.is_backend_compatible()
|
||||
else:
|
||||
return mod._supports_flex_attn
|
||||
|
||||
|
||||
def resolve_transformers_fallback(model_config: ModelConfig,
|
||||
architectures: list[str]):
|
||||
for i, arch in enumerate(architectures):
|
||||
if arch == "TransformersModel":
|
||||
continue
|
||||
custom_module = None
|
||||
auto_map = getattr(model_config.hf_config, "auto_map", None)
|
||||
if auto_map is not None and "AutoModel" in auto_map:
|
||||
custom_module = get_class_from_dynamic_module(
|
||||
model_config.hf_config.auto_map["AutoModel"],
|
||||
model_config.model)
|
||||
# TODO(Isotr0py): Further clean up these raises.
|
||||
# perhaps handled them in _ModelRegistry._raise_for_unsupported?
|
||||
if model_config.model_impl == ModelImpl.TRANSFORMERS:
|
||||
if not is_transformers_impl_compatible(arch, custom_module):
|
||||
raise ValueError(
|
||||
f"The Transformers implementation of {arch} is not "
|
||||
"compatible with vLLM.")
|
||||
architectures[i] = "TransformersModel"
|
||||
if model_config.model_impl == ModelImpl.AUTO:
|
||||
if not is_transformers_impl_compatible(arch, custom_module):
|
||||
raise ValueError(
|
||||
f"{arch} has no vLLM implementation and the Transformers "
|
||||
"implementation is not compatible with vLLM.")
|
||||
logger.warning(
|
||||
"%s has no vLLM implementation, falling back to Transformers "
|
||||
"implementation. Some features may not be supported and "
|
||||
"performance may not be optimal.", arch)
|
||||
architectures[i] = "TransformersModel"
|
||||
return architectures
|
||||
|
||||
|
||||
def get_model_architecture(
|
||||
model_config: ModelConfig) -> Tuple[Type[nn.Module], str]:
|
||||
architectures = getattr(model_config.hf_config, "architectures", [])
|
||||
@@ -38,6 +87,14 @@ def get_model_architecture(
|
||||
and "MixtralForCausalLM" in architectures):
|
||||
architectures = ["QuantMixtralForCausalLM"]
|
||||
|
||||
vllm_supported_archs = ModelRegistry.get_supported_archs()
|
||||
is_vllm_supported = any(arch in vllm_supported_archs
|
||||
for arch in architectures)
|
||||
if (not is_vllm_supported
|
||||
or model_config.model_impl == ModelImpl.TRANSFORMERS):
|
||||
architectures = resolve_transformers_fallback(model_config,
|
||||
architectures)
|
||||
|
||||
model_cls, arch = ModelRegistry.resolve_model_cls(architectures)
|
||||
if model_config.task == "embed":
|
||||
model_cls = as_embedding_model(model_cls)
|
||||
|
||||
Reference in New Issue
Block a user