[Bugfix] Fix ColQwen3.5 pooler projector initialization (#54847)
Signed-off-by: Divakar Verma <divakar.verma@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
46c8a161f4
commit
dbf1a044ea
@@ -121,13 +121,13 @@ class ColQwen3_5Model(
|
|||||||
linear projection layer for per-token embeddings. It supports:
|
linear projection layer for per-token embeddings. It supports:
|
||||||
- "token_embed" task: Per-token embeddings for late interaction scoring
|
- "token_embed" task: Per-token embeddings for late interaction scoring
|
||||||
|
|
||||||
The model produces per-token embeddings by:
|
The model produces L2-normalized per-token embeddings by:
|
||||||
1. Running the Qwen3.5 backbone (vision + language) to get hidden states
|
1. Running the Qwen3.5 backbone (vision + language) to get hidden states
|
||||||
2. Projecting hidden states through a linear layer (hidden_size -> embed_dim)
|
2. Projection and L2-normalization via the pooler (TokenEmbeddingPoolerHead)
|
||||||
3. L2 normalization is handled by the pooler via PoolerNormalize
|
|
||||||
|
|
||||||
Attributes:
|
Attributes:
|
||||||
custom_text_proj: Linear projection from hidden_size to embed_dim
|
custom_text_proj: Linear projection from hidden_size to embed_dim.
|
||||||
|
This is passed to the pooler as pooler.head.projector.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Mark this as a pooling model so vLLM routes to pooler path
|
# Mark this as a pooling model so vLLM routes to pooler path
|
||||||
@@ -187,7 +187,7 @@ class ColQwen3_5Model(
|
|||||||
assert pooler_config is not None
|
assert pooler_config is not None
|
||||||
self.pooler = pooler_for_token_embed(
|
self.pooler = pooler_for_token_embed(
|
||||||
pooler_config,
|
pooler_config,
|
||||||
projector=None,
|
projector=self.custom_text_proj,
|
||||||
)
|
)
|
||||||
|
|
||||||
def forward(
|
def forward(
|
||||||
@@ -198,8 +198,8 @@ class ColQwen3_5Model(
|
|||||||
inputs_embeds: torch.Tensor | None = None,
|
inputs_embeds: torch.Tensor | None = None,
|
||||||
**kwargs: object,
|
**kwargs: object,
|
||||||
) -> torch.Tensor:
|
) -> torch.Tensor:
|
||||||
"""Run forward pass producing per-token embeddings."""
|
"""Run forward pass returning hidden states for pooler."""
|
||||||
hidden_states = super().forward(
|
return super().forward(
|
||||||
input_ids=input_ids,
|
input_ids=input_ids,
|
||||||
positions=positions,
|
positions=positions,
|
||||||
intermediate_tensors=intermediate_tensors,
|
intermediate_tensors=intermediate_tensors,
|
||||||
@@ -207,16 +207,6 @@ class ColQwen3_5Model(
|
|||||||
**kwargs,
|
**kwargs,
|
||||||
)
|
)
|
||||||
|
|
||||||
if not isinstance(hidden_states, torch.Tensor):
|
|
||||||
return hidden_states # type: ignore
|
|
||||||
|
|
||||||
proj_dtype = self.custom_text_proj.weight.dtype
|
|
||||||
if hidden_states.dtype != proj_dtype:
|
|
||||||
hidden_states = hidden_states.to(proj_dtype)
|
|
||||||
|
|
||||||
# Project to embedding dimension (normalization handled by pooler)
|
|
||||||
return self.custom_text_proj(hidden_states)
|
|
||||||
|
|
||||||
# Names used for the projection layer across different ColQwen3.5 variants
|
# Names used for the projection layer across different ColQwen3.5 variants
|
||||||
_PROJ_LAYER_NAMES = {
|
_PROJ_LAYER_NAMES = {
|
||||||
"custom_text_proj", # ColPali naming
|
"custom_text_proj", # ColPali naming
|
||||||
@@ -249,5 +239,8 @@ class ColQwen3_5Model(
|
|||||||
weight = weight.to(device=param.device, dtype=param.dtype)
|
weight = weight.to(device=param.device, dtype=param.dtype)
|
||||||
default_weight_loader(param, weight)
|
default_weight_loader(param, weight)
|
||||||
loaded.add(f"custom_text_proj.{param_name}")
|
loaded.add(f"custom_text_proj.{param_name}")
|
||||||
|
# Also mark as loaded under pooler path since custom_text_proj
|
||||||
|
# is assigned to pooler.head.projector
|
||||||
|
loaded.add(f"pooler.head.projector.{param_name}")
|
||||||
|
|
||||||
return loaded
|
return loaded
|
||||||
|
|||||||
Reference in New Issue
Block a user