[Model] Support VLMs with transformers backend (#20543)
Signed-off-by: raushan <raushan@huggingface.co> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
Cyrus Leung
parent
51ba839555
commit
9499e26e2a
+30
-9
@@ -562,6 +562,10 @@ class ModelConfig:
|
||||
|
||||
self.task = "embed"
|
||||
|
||||
model_info, arch = self.registry.inspect_model_cls(self.architectures)
|
||||
self._model_info = model_info
|
||||
self._architecture = arch
|
||||
|
||||
all_supported_tasks = self._get_supported_tasks(self.task)
|
||||
logger.debug("Tasks supported by runner type: %s", all_supported_tasks)
|
||||
supported_runner_types = self._get_supported_runner_types(
|
||||
@@ -587,10 +591,6 @@ class ModelConfig:
|
||||
else:
|
||||
self.truncation_side = "right"
|
||||
|
||||
model_info, arch = self.registry.inspect_model_cls(self.architectures)
|
||||
self._model_info = model_info
|
||||
self._architecture = arch
|
||||
|
||||
self.pooler_config = self._init_pooler_config()
|
||||
|
||||
self.dtype = _get_and_verify_dtype(
|
||||
@@ -674,6 +674,16 @@ class ModelConfig:
|
||||
"max_model_len must be an integer after __post_init__.")
|
||||
return self
|
||||
|
||||
def _get_transformers_backend_cls(self) -> str:
|
||||
"""Determine which Transformers backend class will be used if
|
||||
`model_impl` is set to `transformers` or `auto`."""
|
||||
if self.hf_config != self.hf_text_config:
|
||||
# If 'hf_text_config' is the same as 'hf_config'. If not, it is
|
||||
# probably a composite config, i.e. multimodal
|
||||
return "TransformersForMultimodalLM"
|
||||
else:
|
||||
return "TransformersForCausalLM"
|
||||
|
||||
@property
|
||||
def registry(self):
|
||||
return me_models.ModelRegistry
|
||||
@@ -681,7 +691,19 @@ class ModelConfig:
|
||||
@property
|
||||
def architectures(self) -> list[str]:
|
||||
# architectures in the model config.
|
||||
return getattr(self.hf_config, "architectures", [])
|
||||
architectures = getattr(self.hf_config, "architectures", [])
|
||||
# The registry assumes that it can always inspect the vLLM model class
|
||||
# for a given architecture. This assumption breaks down for the
|
||||
# Transformers backend, which may use a different class depending on
|
||||
# the model type. To work around this, we add the correct Transformers
|
||||
# backend class to the architectures list. We must do this here because
|
||||
# we need access to the `hf_config` to determine the backend class.
|
||||
transformers_backend_cls = self._get_transformers_backend_cls()
|
||||
if (self.model_impl != ModelImpl.VLLM.value
|
||||
and all(arch != transformers_backend_cls
|
||||
for arch in architectures)):
|
||||
architectures.append(transformers_backend_cls)
|
||||
return architectures
|
||||
|
||||
@property
|
||||
def architecture(self) -> str:
|
||||
@@ -827,10 +849,9 @@ class ModelConfig:
|
||||
("EmbeddingModel", "embed"),
|
||||
("RewardModel", "reward"),
|
||||
]
|
||||
_, arch = self.registry.inspect_model_cls(architectures)
|
||||
|
||||
for suffix, pref_task in suffix_to_preferred_task:
|
||||
if arch.endswith(suffix):
|
||||
if self.architecture.endswith(suffix):
|
||||
return pref_task
|
||||
|
||||
return "embed"
|
||||
@@ -944,10 +965,10 @@ class ModelConfig:
|
||||
("EmbeddingModel", "pooling"),
|
||||
("RewardModel", "pooling"),
|
||||
]
|
||||
_, arch = self.registry.inspect_model_cls(self.architectures)
|
||||
|
||||
for suffix, pref_runner in suffix_to_preferred_runner:
|
||||
if arch.endswith(suffix) and pref_runner in supported_runner_types:
|
||||
if self.architecture.endswith(
|
||||
suffix) and pref_runner in supported_runner_types:
|
||||
return pref_runner
|
||||
|
||||
if "generate" in supported_runner_types:
|
||||
|
||||
Reference in New Issue
Block a user