[Model] Support VLMs with transformers backend (#20543)
Signed-off-by: raushan <raushan@huggingface.co> Signed-off-by: Isotr0py <2037008807@qq.com> Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Isotr0py <2037008807@qq.com> Co-authored-by: Isotr0py <mozf@mail2.sysu.edu.cn> Co-authored-by: Cyrus Leung <cyrus.tl.leung@gmail.com>
This commit is contained in:
co-authored by
Isotr0py
Isotr0py
Cyrus Leung
parent
51ba839555
commit
9499e26e2a
@@ -253,6 +253,7 @@ _SPECULATIVE_DECODING_MODELS = {
|
||||
}
|
||||
|
||||
_TRANSFORMERS_MODELS = {
|
||||
"TransformersForMultimodalLM": ("transformers", "TransformersForMultimodalLM"), # noqa: E501
|
||||
"TransformersForCausalLM": ("transformers", "TransformersForCausalLM"),
|
||||
}
|
||||
# yapf: enable
|
||||
@@ -504,9 +505,14 @@ class _ModelRegistry:
|
||||
if causal_lm_arch in self.models:
|
||||
normalized_arch.append(arch)
|
||||
|
||||
# make sure Transformers backend is put at the last as a fallback
|
||||
if len(normalized_arch) != len(architectures):
|
||||
normalized_arch.append("TransformersForCausalLM")
|
||||
# NOTE(Isotr0py): Be careful of architectures' order!
|
||||
# Make sure Transformers backend architecture is at the end of the
|
||||
# list, otherwise pooling models automatic conversion will fail!
|
||||
for arch in normalized_arch:
|
||||
if arch.startswith("TransformersFor"):
|
||||
normalized_arch.remove(arch)
|
||||
normalized_arch.append(arch)
|
||||
|
||||
return normalized_arch
|
||||
|
||||
def inspect_model_cls(
|
||||
|
||||
Reference in New Issue
Block a user