[Model] Add ColPali late interaction model for multi-modal retrieval (#36818)
Signed-off-by: Nikita Sukharev <kaonael@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
@@ -33,6 +33,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
|
||||
"blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
|
||||
"chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
"clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
"colpali": CHAT_TEMPLATES_DIR / "template_basic.jinja",
|
||||
"deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
|
||||
"deepseek_ocr2": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
|
||||
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
|
||||
|
||||
@@ -78,6 +78,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
bagel="BagelConfig",
|
||||
chatglm="ChatGLMConfig",
|
||||
colmodernvbert="ColModernVBertConfig",
|
||||
colpali="ColPaliConfig",
|
||||
colqwen3="ColQwen3Config",
|
||||
ops_colqwen3="OpsColQwen3Config",
|
||||
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
|
||||
|
||||
@@ -20,6 +20,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"BagelConfig": "vllm.transformers_utils.configs.bagel",
|
||||
"ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
|
||||
"ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
|
||||
"ColPaliConfig": "vllm.transformers_utils.configs.colpali",
|
||||
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
|
||||
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
|
||||
@@ -76,6 +77,7 @@ __all__ = [
|
||||
"BagelConfig",
|
||||
"ChatGLMConfig",
|
||||
"ColModernVBertConfig",
|
||||
"ColPaliConfig",
|
||||
"ColQwen3Config",
|
||||
"OpsColQwen3Config",
|
||||
"Qwen3VLNemotronEmbedConfig",
|
||||
|
||||
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
ColPali configuration that extends PaliGemmaConfig with embedding projection
|
||||
fields. This allows ColPali models to be loaded without trust_remote_code
|
||||
by mapping their custom model_type (colpali) to a standard config class
|
||||
that vLLM understands.
|
||||
|
||||
Supported model_types:
|
||||
- colpali (vidore/colpali-v1.3-hf)
|
||||
"""
|
||||
|
||||
from transformers import PaliGemmaConfig
|
||||
|
||||
|
||||
class ColPaliConfig(PaliGemmaConfig):
|
||||
"""Configuration class for ColPali models.
|
||||
|
||||
Extends PaliGemmaConfig with additional fields used by ColPali variants
|
||||
for the embedding projection layer.
|
||||
"""
|
||||
|
||||
model_type = "colpali"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
embedding_dim: int | None = None,
|
||||
embed_dim: int | None = None,
|
||||
dim: int | None = None,
|
||||
projection_dim: int | None = None,
|
||||
colbert_dim: int | None = None,
|
||||
pooling: str | None = None,
|
||||
vlm_config: dict | None = None,
|
||||
**kwargs,
|
||||
):
|
||||
# Store embedding projection config fields
|
||||
self.embedding_dim = embedding_dim
|
||||
self.embed_dim = embed_dim
|
||||
self.dim = dim
|
||||
self.projection_dim = projection_dim
|
||||
self.colbert_dim = colbert_dim
|
||||
self.pooling = pooling
|
||||
|
||||
# The HF checkpoint nests PaliGemma config inside "vlm_config".
|
||||
# Flatten it so PaliGemmaConfig receives vision_config, text_config,
|
||||
# image_token_index, etc. directly.
|
||||
# Use setdefault to avoid overwriting keys already set (e.g.
|
||||
# model_type="colpali" would be clobbered by "paligemma" from
|
||||
# vlm_config).
|
||||
if vlm_config is not None:
|
||||
vlm_dict = (
|
||||
vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict()
|
||||
)
|
||||
_conflicting = {"model_type", "_name_or_path"}
|
||||
for key, value in vlm_dict.items():
|
||||
if key not in _conflicting:
|
||||
kwargs.setdefault(key, value)
|
||||
|
||||
super().__init__(**kwargs)
|
||||
Reference in New Issue
Block a user