[Model] Add support for the multi-modal Llama 3.2 model (#8811)
Co-authored-by: simon-mo <xmo@berkeley.edu> Co-authored-by: Chang Su <chang.s.su@oracle.com> Co-authored-by: Simon Mo <simon.mo@hey.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
co-authored by
simon-mo
Chang Su
Simon Mo
Roger Wang
Roger Wang
parent
4f1ba0844b
commit
770ec6024f
@@ -22,9 +22,10 @@ from vllm.transformers_utils.configs import (ChatGLMConfig, DbrxConfig,
|
||||
EAGLEConfig, ExaoneConfig,
|
||||
GraniteConfig, InternVLChatConfig,
|
||||
JAISConfig, MedusaConfig,
|
||||
MLPSpeculatorConfig, MPTConfig,
|
||||
NemotronConfig, RWConfig,
|
||||
SolarConfig, UltravoxConfig)
|
||||
MllamaConfig, MLPSpeculatorConfig,
|
||||
MPTConfig, NemotronConfig,
|
||||
RWConfig, SolarConfig,
|
||||
UltravoxConfig)
|
||||
# yapf: enable
|
||||
from vllm.transformers_utils.utils import check_gguf_file
|
||||
|
||||
@@ -37,6 +38,10 @@ MISTRAL_CONFIG_NAME = "params.json"
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
_CONFIG_REGISTRY_OVERRIDE_HF: Dict[str, Type[PretrainedConfig]] = {
|
||||
"mllama": MllamaConfig
|
||||
}
|
||||
|
||||
_CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
|
||||
"chatglm": ChatGLMConfig,
|
||||
"dbrx": DbrxConfig,
|
||||
@@ -55,11 +60,15 @@ _CONFIG_REGISTRY: Dict[str, Type[PretrainedConfig]] = {
|
||||
# Granite can be removed from here once we have upgraded to
|
||||
# transformers 4.45+
|
||||
"granite": GraniteConfig,
|
||||
**_CONFIG_REGISTRY_OVERRIDE_HF
|
||||
}
|
||||
|
||||
for name, cls in _CONFIG_REGISTRY.items():
|
||||
with contextlib.suppress(ValueError):
|
||||
AutoConfig.register(name, cls)
|
||||
if name in _CONFIG_REGISTRY_OVERRIDE_HF:
|
||||
AutoConfig.register(name, cls, exist_ok=True)
|
||||
else:
|
||||
AutoConfig.register(name, cls)
|
||||
|
||||
|
||||
class ConfigFormat(str, enum.Enum):
|
||||
|
||||
@@ -10,6 +10,7 @@ from vllm.transformers_utils.configs.granite import GraniteConfig
|
||||
from vllm.transformers_utils.configs.internvl import InternVLChatConfig
|
||||
from vllm.transformers_utils.configs.jais import JAISConfig
|
||||
from vllm.transformers_utils.configs.medusa import MedusaConfig
|
||||
from vllm.transformers_utils.configs.mllama import MllamaConfig
|
||||
from vllm.transformers_utils.configs.mlp_speculator import MLPSpeculatorConfig
|
||||
from vllm.transformers_utils.configs.mpt import MPTConfig
|
||||
from vllm.transformers_utils.configs.nemotron import NemotronConfig
|
||||
@@ -26,6 +27,7 @@ __all__ = [
|
||||
"MedusaConfig",
|
||||
"EAGLEConfig",
|
||||
"ExaoneConfig",
|
||||
"MllamaConfig",
|
||||
"MLPSpeculatorConfig",
|
||||
"NemotronConfig",
|
||||
"SolarConfig",
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
from transformers.models.mllama import configuration_mllama as mllama_hf_config
|
||||
|
||||
|
||||
class MllamaTextConfig(mllama_hf_config.MllamaTextConfig):
|
||||
'''
|
||||
Use this class to override is_encoder_decoder:
|
||||
- transformers regards mllama as is_encoder_decoder=False
|
||||
- vllm needs is_encoder_decoder=True to enable cross-attention
|
||||
'''
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
**kwargs,
|
||||
):
|
||||
super().__init__(**kwargs)
|
||||
self.is_encoder_decoder = True
|
||||
|
||||
|
||||
class MllamaConfig(mllama_hf_config.MllamaConfig):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
**kwargs,
|
||||
):
|
||||
if isinstance(text_config, dict):
|
||||
text_config = MllamaTextConfig(**text_config)
|
||||
super().__init__(text_config=text_config, **kwargs)
|
||||
@@ -111,7 +111,6 @@ def get_tokenizer(
|
||||
'encoding and decoding.',
|
||||
FutureWarning,
|
||||
stacklevel=2)
|
||||
|
||||
if tokenizer_mode == "mistral":
|
||||
tokenizer = MistralTokenizer.from_pretrained(str(tokenizer_name),
|
||||
revision=revision)
|
||||
|
||||
Reference in New Issue
Block a user