From dbf1a044eacdd5a5a063bab3701e63549ac87a8e Mon Sep 17 00:00:00 2001 From: Divakar Verma <137818590+divakar-amd@users.noreply.github.com> Date: Tue, 1 Sep 2026 22:35:43 -0400 Subject: [PATCH] [Bugfix] Fix ColQwen3.5 pooler projector initialization (#54847) Signed-off-by: Divakar Verma Co-authored-by: Andreas Karatzas --- vllm/model_executor/models/colqwen3_5.py | 27 +++++++++--------------- 1 file changed, 10 insertions(+), 17 deletions(-) diff --git a/vllm/model_executor/models/colqwen3_5.py b/vllm/model_executor/models/colqwen3_5.py index 68ec623835..dabb4885ee 100644 --- a/vllm/model_executor/models/colqwen3_5.py +++ b/vllm/model_executor/models/colqwen3_5.py @@ -121,13 +121,13 @@ class ColQwen3_5Model( linear projection layer for per-token embeddings. It supports: - "token_embed" task: Per-token embeddings for late interaction scoring - The model produces per-token embeddings by: + The model produces L2-normalized per-token embeddings by: 1. Running the Qwen3.5 backbone (vision + language) to get hidden states - 2. Projecting hidden states through a linear layer (hidden_size -> embed_dim) - 3. L2 normalization is handled by the pooler via PoolerNormalize + 2. Projection and L2-normalization via the pooler (TokenEmbeddingPoolerHead) Attributes: - custom_text_proj: Linear projection from hidden_size to embed_dim + custom_text_proj: Linear projection from hidden_size to embed_dim. + This is passed to the pooler as pooler.head.projector. """ # Mark this as a pooling model so vLLM routes to pooler path @@ -187,7 +187,7 @@ class ColQwen3_5Model( assert pooler_config is not None self.pooler = pooler_for_token_embed( pooler_config, - projector=None, + projector=self.custom_text_proj, ) def forward( @@ -198,8 +198,8 @@ class ColQwen3_5Model( inputs_embeds: torch.Tensor | None = None, **kwargs: object, ) -> torch.Tensor: - """Run forward pass producing per-token embeddings.""" - hidden_states = super().forward( + """Run forward pass returning hidden states for pooler.""" + return super().forward( input_ids=input_ids, positions=positions, intermediate_tensors=intermediate_tensors, @@ -207,16 +207,6 @@ class ColQwen3_5Model( **kwargs, ) - if not isinstance(hidden_states, torch.Tensor): - return hidden_states # type: ignore - - proj_dtype = self.custom_text_proj.weight.dtype - if hidden_states.dtype != proj_dtype: - hidden_states = hidden_states.to(proj_dtype) - - # Project to embedding dimension (normalization handled by pooler) - return self.custom_text_proj(hidden_states) - # Names used for the projection layer across different ColQwen3.5 variants _PROJ_LAYER_NAMES = { "custom_text_proj", # ColPali naming @@ -249,5 +239,8 @@ class ColQwen3_5Model( weight = weight.to(device=param.device, dtype=param.dtype) default_weight_loader(param, weight) loaded.add(f"custom_text_proj.{param_name}") + # Also mark as loaded under pooler path since custom_text_proj + # is assigned to pooler.head.projector + loaded.add(f"pooler.head.projector.{param_name}") return loaded