[Model] Add LFM2-ColBERT-350M support (#37528)
Signed-off-by: Ilya Boytsov <ilyaboytsov1805@gmail.com>
This commit is contained in:
@@ -27,8 +27,9 @@ from vllm.model_executor.layers.pooler import Pooler
|
||||
from vllm.model_executor.layers.pooler.tokwise import pooler_for_token_embed
|
||||
|
||||
from .bert import BertEmbeddingModel, BertModel
|
||||
from .interfaces import SupportsLateInteraction
|
||||
from .interfaces import HasInnerState, IsHybrid, SupportsLateInteraction
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .lfm2 import Lfm2ForCausalLM, Lfm2Model
|
||||
|
||||
|
||||
class ColBERTMixin(nn.Module, SupportsLateInteraction):
|
||||
@@ -414,3 +415,98 @@ class ColBERTJinaRobertaModel(ColBERTMixin, nn.Module):
|
||||
loaded.update(colbert_loaded)
|
||||
|
||||
return loaded
|
||||
|
||||
|
||||
# -----------------------------------------------------------------------
|
||||
# Concrete model: ColBERT + LFM2 backbone
|
||||
# -----------------------------------------------------------------------
|
||||
|
||||
|
||||
@default_pooling_type(seq_pooling_type="CLS", tok_pooling_type="ALL")
|
||||
class ColBERTLfm2Model(ColBERTMixin, nn.Module, HasInnerState, IsHybrid):
|
||||
"""ColBERT late interaction model with LFM2 backbone.
|
||||
|
||||
For ``LiquidAI/LFM2-ColBERT-350M`` and similar models.
|
||||
|
||||
The projection is auto-loaded from sentence-transformers ``1_Dense/``
|
||||
when not present in the main checkpoint.
|
||||
"""
|
||||
|
||||
is_pooling_model = True
|
||||
# LFM2 is a hybrid model (attention + SSM layers); these flags ensure
|
||||
# HybridAttentionMambaModelConfig.verify_and_update_config runs so that
|
||||
# mamba_block_size and related cache settings are correctly initialised.
|
||||
is_hybrid = True
|
||||
has_inner_state = True
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_shape_from_config(cls, vllm_config: VllmConfig):
|
||||
return Lfm2ForCausalLM.get_mamba_state_shape_from_config(vllm_config)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_dtype_from_config(cls, vllm_config: VllmConfig):
|
||||
return Lfm2ForCausalLM.get_mamba_state_dtype_from_config(vllm_config)
|
||||
|
||||
@classmethod
|
||||
def get_mamba_state_copy_func(cls):
|
||||
return Lfm2ForCausalLM.get_mamba_state_copy_func()
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.model_config.hf_config
|
||||
|
||||
colbert_dim = self.get_colbert_dim_from_config(config)
|
||||
self._init_colbert_components(
|
||||
hidden_size=config.hidden_size,
|
||||
colbert_dim=colbert_dim,
|
||||
head_dtype=vllm_config.model_config.head_dtype,
|
||||
)
|
||||
|
||||
self.model = Lfm2Model(
|
||||
vllm_config=vllm_config,
|
||||
prefix=prefix,
|
||||
)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
self.pooler = self._build_colbert_pooler(pooler_config)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
intermediate_tensors=None,
|
||||
inputs_embeds: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return self.model(
|
||||
input_ids=input_ids,
|
||||
positions=positions,
|
||||
inputs_embeds=inputs_embeds,
|
||||
intermediate_tensors=intermediate_tensors,
|
||||
)
|
||||
|
||||
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]):
|
||||
other_weights, colbert_loaded = self._load_colbert_weights(weights)
|
||||
|
||||
# Strip "model." prefix added by the embedding adapter
|
||||
model_weights = [
|
||||
(n[len("model.") :] if n.startswith("model.") else n, w)
|
||||
for n, w in other_weights
|
||||
]
|
||||
loaded_model = self.model.load_weights(model_weights)
|
||||
loaded = {f"model.{name}" for name in loaded_model} | colbert_loaded
|
||||
|
||||
# When the ST projector was auto-loaded during init
|
||||
# (not from the main checkpoint), mark its params as loaded
|
||||
# so the weight validator doesn't complain.
|
||||
if hasattr(self.pooler, "head"):
|
||||
head = self.pooler.head
|
||||
projector = getattr(head, "projector", None)
|
||||
if projector is not None and isinstance(projector, nn.Module):
|
||||
for name, _ in projector.named_parameters():
|
||||
loaded.add(f"pooler.head.projector.{name}")
|
||||
|
||||
return loaded
|
||||
|
||||
@@ -269,6 +269,7 @@ _LATE_INTERACTION_MODELS = {
|
||||
"HF_ColBERT": ("colbert", "ColBERTModel"),
|
||||
"ColBERTModernBertModel": ("colbert", "ColBERTModernBertModel"),
|
||||
"ColBERTJinaRobertaModel": ("colbert", "ColBERTJinaRobertaModel"),
|
||||
"ColBERTLfm2Model": ("colbert", "ColBERTLfm2Model"),
|
||||
# [Multimodal]
|
||||
"ColModernVBertForRetrieval": ("colmodernvbert", "ColModernVBertForRetrieval"),
|
||||
"ColPaliForRetrieval": ("colpali", "ColPaliModel"),
|
||||
|
||||
Reference in New Issue
Block a user