[Multimodal] Automatically select registered video loader for VLM (#44126)

Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
Isotr0py
2026-06-02 09:09:47 +00:00
committed by GitHub
parent e30313220c
commit f8e9c56d15
5 changed files with 188 additions and 9 deletions
+38
View File
@@ -161,6 +161,44 @@ def get_processor_cls_name_from_config(
return None
def get_video_processor_cls_name_from_config(
processor_name: str,
revision: str | None = "main",
) -> str | None:
processor_name = convert_model_repo_to_path(processor_name)
config_file = [
"video_preprocessor_config.json",
"preprocessor_config.json",
]
for file in config_file:
config = get_hf_file_to_dict(file, processor_name, revision=revision)
if config and "video_processor_type" in config:
return config["video_processor_type"]
return None
_cached_get_video_processor_cls_name = lru_cache(
get_video_processor_cls_name_from_config
)
def get_video_processor_cls_name(
model_config: "ModelConfig",
) -> str | None:
if is_gguf(model_config.model):
assert not is_gguf(model_config.tokenizer), (
"For multimodal GGUF models, the original tokenizer "
"should be used to correctly load video processor metadata."
)
model = model_config.tokenizer
revision = model_config.tokenizer_revision
else:
model = model_config.model
revision = model_config.revision
return _cached_get_video_processor_cls_name(model, revision=revision)
def get_processor(
processor_name: str,
*args: Any,