[Bugfix] Fix ColQwen3.5 pooler projector initialization (#54847)

Signed-off-by: Divakar Verma <divakar.verma@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
Divakar Verma
2026-09-02 10:35:43 +08:00
committed by GitHub
co-authored by Andreas Karatzas
parent 46c8a161f4
commit dbf1a044ea
+10 -17
View File
@@ -121,13 +121,13 @@ class ColQwen3_5Model(
linear projection layer for per-token embeddings. It supports: linear projection layer for per-token embeddings. It supports:
- "token_embed" task: Per-token embeddings for late interaction scoring - "token_embed" task: Per-token embeddings for late interaction scoring
The model produces per-token embeddings by: The model produces L2-normalized per-token embeddings by:
1. Running the Qwen3.5 backbone (vision + language) to get hidden states 1. Running the Qwen3.5 backbone (vision + language) to get hidden states
2. Projecting hidden states through a linear layer (hidden_size -> embed_dim) 2. Projection and L2-normalization via the pooler (TokenEmbeddingPoolerHead)
3. L2 normalization is handled by the pooler via PoolerNormalize
Attributes: Attributes:
custom_text_proj: Linear projection from hidden_size to embed_dim custom_text_proj: Linear projection from hidden_size to embed_dim.
This is passed to the pooler as pooler.head.projector.
""" """
# Mark this as a pooling model so vLLM routes to pooler path # Mark this as a pooling model so vLLM routes to pooler path
@@ -187,7 +187,7 @@ class ColQwen3_5Model(
assert pooler_config is not None assert pooler_config is not None
self.pooler = pooler_for_token_embed( self.pooler = pooler_for_token_embed(
pooler_config, pooler_config,
projector=None, projector=self.custom_text_proj,
) )
def forward( def forward(
@@ -198,8 +198,8 @@ class ColQwen3_5Model(
inputs_embeds: torch.Tensor | None = None, inputs_embeds: torch.Tensor | None = None,
**kwargs: object, **kwargs: object,
) -> torch.Tensor: ) -> torch.Tensor:
"""Run forward pass producing per-token embeddings.""" """Run forward pass returning hidden states for pooler."""
hidden_states = super().forward( return super().forward(
input_ids=input_ids, input_ids=input_ids,
positions=positions, positions=positions,
intermediate_tensors=intermediate_tensors, intermediate_tensors=intermediate_tensors,
@@ -207,16 +207,6 @@ class ColQwen3_5Model(
**kwargs, **kwargs,
) )
if not isinstance(hidden_states, torch.Tensor):
return hidden_states # type: ignore
proj_dtype = self.custom_text_proj.weight.dtype
if hidden_states.dtype != proj_dtype:
hidden_states = hidden_states.to(proj_dtype)
# Project to embedding dimension (normalization handled by pooler)
return self.custom_text_proj(hidden_states)
# Names used for the projection layer across different ColQwen3.5 variants # Names used for the projection layer across different ColQwen3.5 variants
_PROJ_LAYER_NAMES = { _PROJ_LAYER_NAMES = {
"custom_text_proj", # ColPali naming "custom_text_proj", # ColPali naming
@@ -249,5 +239,8 @@ class ColQwen3_5Model(
weight = weight.to(device=param.device, dtype=param.dtype) weight = weight.to(device=param.device, dtype=param.dtype)
default_weight_loader(param, weight) default_weight_loader(param, weight)
loaded.add(f"custom_text_proj.{param_name}") loaded.add(f"custom_text_proj.{param_name}")
# Also mark as loaded under pooler path since custom_text_proj
# is assigned to pooler.head.projector
loaded.add(f"pooler.head.projector.{param_name}")
return loaded return loaded