[Model] ColQwen3.5: fix retrieval correctness (bias + bidirectional) (#46108)

Signed-off-by: Athrael Soju <athrael.soju@gmail.com>
Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Athrael Soju
2026-06-22 17:25:54 +08:00
committed by GitHub
co-authored by Claude Opus 4.8
parent cec2ec1176
commit 3c8e49596c
7 changed files with 79 additions and 5 deletions
+9 -1
View File
@@ -15,6 +15,7 @@ Based on: Qwen3.5 backbone with custom text projection
Target models:
- athrael-soju/colqwen3.5-4.5B-v3
- vultr/VultronRetrieverPrime-Qwen3.5-8B
"""
from collections.abc import Iterable, Mapping
@@ -166,12 +167,19 @@ class ColQwen3_5Model(
or 128 # default from reference implementation
)
# ColPali defines `custom_text_proj = nn.Linear(hidden, dim)`, i.e.
# bias=True by default, and the trained ColQwen3.5 checkpoints ship a
# `custom_text_proj.bias`. Construct with a bias and zero-initialize it:
# a (legacy) bias-less checkpoint then behaves identically to bias=False,
# while load_weights() below picks up a trained bias instead of silently
# dropping it (which shifts every per-token vector and the MaxSim ranking).
self.custom_text_proj = nn.Linear(
hidden_size,
self.embed_dim,
bias=False,
bias=True,
dtype=head_dtype,
)
nn.init.zeros_(self.custom_text_proj.bias)
pooler_config = vllm_config.model_config.pooler_config
assert pooler_config is not None
+15 -1
View File
@@ -627,6 +627,20 @@ class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
)
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
"""ColQwen3.5 (late-interaction retrieval) inherits Qwen3.5's mamba cache
handling and additionally serves BIDIRECTIONAL attention: ColPali-style
document/query encoding attends over the whole sequence, not causally. Set
is_causal=False so Qwen3NextAttention builds its full_attention layers with
AttentionType.ENCODER_ONLY (the linear_attention GatedDeltaNet layers are
unaffected). Generation arches keep the parent (causal) and are untouched.
"""
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
model_config.hf_config.is_causal = False
class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
@staticmethod
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
@@ -656,7 +670,7 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
"ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
"ColQwen3_5": ColQwen3_5Config,
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
+11
View File
@@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
from vllm.v1.attention.backend import AttentionType
from .interfaces import (
EagleModelMixin,
@@ -267,6 +268,15 @@ class Qwen3NextAttention(nn.Module):
dual_chunk_attention_config=self.dual_chunk_attention_config,
)
# Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL
# attention on the full_attention layers; they set config.is_causal=False
# via a VerifyAndUpdateConfig handler. Generation models leave is_causal
# unset (-> causal/DECODER), so this is a no-op for them. Mirrors qwen3.py.
attn_type = (
AttentionType.DECODER
if getattr(config, "is_causal", True)
else AttentionType.ENCODER_ONLY
)
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -275,6 +285,7 @@ class Qwen3NextAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
attn_type=attn_type,
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": self.dual_chunk_attention_config,