[New Model] Add ColModernVBERT (#34558)

Signed-off-by: Athrael Soju <athrael.soju@gmail.com>
Signed-off-by: athrael-soju <athrael-soju@users.noreply.github.com>
This commit is contained in:
Athrael Soju
2026-02-22 12:23:41 +08:00
committed by GitHub
parent d24bdd7c4b
commit 970861ac0c
9 changed files with 784 additions and 0 deletions
+1
View File
@@ -74,6 +74,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
afmoe="AfmoeConfig",
bagel="BagelConfig",
chatglm="ChatGLMConfig",
colmodernvbert="ColModernVBertConfig",
colqwen3="ColQwen3Config",
ops_colqwen3="OpsColQwen3Config",
qwen3_vl_nemotron_embed="Qwen3VLNemotronEmbedConfig",
@@ -18,6 +18,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
"AfmoeConfig": "vllm.transformers_utils.configs.afmoe",
"BagelConfig": "vllm.transformers_utils.configs.bagel",
"ChatGLMConfig": "vllm.transformers_utils.configs.chatglm",
"ColModernVBertConfig": "vllm.transformers_utils.configs.colmodernvbert",
"ColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"OpsColQwen3Config": "vllm.transformers_utils.configs.colqwen3",
"Qwen3VLNemotronEmbedConfig": "vllm.transformers_utils.configs.colqwen3",
@@ -71,6 +72,7 @@ __all__ = [
"AfmoeConfig",
"BagelConfig",
"ChatGLMConfig",
"ColModernVBertConfig",
"ColQwen3Config",
"OpsColQwen3Config",
"Qwen3VLNemotronEmbedConfig",
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Configuration for ColModernVBERT visual document retrieval model.
ColModernVBERT combines SigLIP vision encoder + ModernBERT text encoder
with a pixel shuffle connector and ColBERT-style 128-dim per-token embeddings.
Reference: https://huggingface.co/ModernVBERT/colmodernvbert-merged
"""
from transformers import ModernBertConfig, PretrainedConfig, SiglipVisionConfig
class ColModernVBertConfig(PretrainedConfig):
model_type = "colmodernvbert"
def __init__(
self,
embedding_dim: int = 128,
vlm_config: dict | None = None,
**kwargs,
):
super().__init__(**kwargs)
self.embedding_dim = embedding_dim
if vlm_config is None:
vlm_config = {}
# Top-level VLM fields
self.image_token_id = vlm_config.get("image_token_id", 50407)
self.pixel_shuffle_factor = vlm_config.get("pixel_shuffle_factor", 4)
self.hidden_size = vlm_config.get("hidden_size", 768)
additional_vocab_size = vlm_config.get("additional_vocab_size", 40)
# Text config (ModernBERT)
text_cfg = vlm_config.get("text_config", {})
base_vocab = text_cfg.get("vocab_size", 50368)
self.text_config = ModernBertConfig(
vocab_size=base_vocab + additional_vocab_size,
hidden_size=text_cfg.get("hidden_size", 768),
intermediate_size=text_cfg.get("intermediate_size", 1152),
num_hidden_layers=text_cfg.get("num_hidden_layers", 22),
num_attention_heads=text_cfg.get("num_attention_heads", 12),
mlp_bias=text_cfg.get("mlp_bias", False),
max_position_embeddings=vlm_config.get("max_position_embeddings", 8192),
)
# Vision config (SigLIP)
vis_cfg = vlm_config.get("vision_config", {})
self.vision_config = SiglipVisionConfig(
hidden_size=vis_cfg.get("embed_dim", 768),
image_size=vis_cfg.get("image_size", 512),
patch_size=vis_cfg.get("patch_size", 16),
num_hidden_layers=vis_cfg.get("num_hidden_layers", 12),
intermediate_size=vis_cfg.get("intermediate_size", 3072),
num_attention_heads=vis_cfg.get("num_attention_heads", 12),
)
@property
def image_seq_len(self) -> int:
ps = self.vision_config.image_size // self.vision_config.patch_size
return (ps * ps) // (self.pixel_shuffle_factor**2)
def get_text_config(self, **kwargs):
return self.text_config