[Model] Add ColPali late interaction model for multi-modal retrieval (#36818)

Signed-off-by: Nikita Sukharev <kaonael@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Nikita
2026-03-13 02:18:57 +00:00
committed by GitHub
co-authored by Cyrus Leung
parent 5e1a373d2e
commit 10f08dedfa
9 changed files with 634 additions and 0 deletions
@@ -33,6 +33,7 @@ _MODEL_TYPE_TO_CHAT_TEMPLATE_FALLBACK: dict[str, ChatTemplatePath] = {
"blip-2": CHAT_TEMPLATES_DIR / "template_blip2.jinja",
"chameleon": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"clip": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"colpali": CHAT_TEMPLATES_DIR / "template_basic.jinja",
"deepseek_ocr": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
"deepseek_ocr2": CHAT_TEMPLATES_DIR / "template_deepseek_ocr.jinja",
"deepseek_vl_v2": CHAT_TEMPLATES_DIR / "template_deepseek_vl2.jinja",
+1
View File
@@ -78,6 +78,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
bagel="BagelConfig",
chatglm="ChatGLMConfig",
colmodernvbert="ColModernVBertConfig",
colpali="ColPaliConfig",
colqwen3="ColQwen3Config",
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
@@ -20,6 +20,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"BagelConfig": "vllm.transformers_utils.configs.bagel",
"ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
"ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
"ColPaliConfig": "vllm.transformers_utils.configs.colpali",
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
@@ -76,6 +77,7 @@ __all__ = [
"BagelConfig",
"ChatGLMConfig",
"ColModernVBertConfig",
"ColPaliConfig",
"ColQwen3Config",
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
@@ -0,0 +1,59 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""
ColPali configuration that extends PaliGemmaConfig with embedding projection
fields. This allows ColPali models to be loaded without trust_remote_code
by mapping their custom model_type (colpali) to a standard config class
that vLLM understands.
Supported model_types:
- colpali (vidore/colpali-v1.3-hf)
"""
from transformers import PaliGemmaConfig
class ColPaliConfig(PaliGemmaConfig):
"""Configuration class for ColPali models.
Extends PaliGemmaConfig with additional fields used by ColPali variants
for the embedding projection layer.
"""
model_type = "colpali"
def __init__(
self,
embedding_dim: int | None = None,
embed_dim: int | None = None,
dim: int | None = None,
projection_dim: int | None = None,
colbert_dim: int | None = None,
pooling: str | None = None,
vlm_config: dict | None = None,
**kwargs,
):
# Store embedding projection config fields
self.embedding_dim = embedding_dim
self.embed_dim = embed_dim
self.dim = dim
self.projection_dim = projection_dim
self.colbert_dim = colbert_dim
self.pooling = pooling
# The HF checkpoint nests PaliGemma config inside "vlm_config".
# Flatten it so PaliGemmaConfig receives vision_config, text_config,
# image_token_index, etc. directly.
# Use setdefault to avoid overwriting keys already set (e.g.
# model_type="colpali" would be clobbered by "paligemma" from
# vlm_config).
if vlm_config is not None:
vlm_dict = (
vlm_config if isinstance(vlm_config, dict) else vlm_config.to_dict()
)
_conflicting = {"model_type", "_name_or_path"}
for key, value in vlm_dict.items():
if key not in _conflicting:
kwargs.setdefault(key, value)
super().__init__(**kwargs)