[Model] ColQwen3.5: fix retrieval correctness (bias + bidirectional) (#46108)
Signed-off-by: Athrael Soju <athrael.soju@gmail.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
cec2ec1176
commit
3c8e49596c
@@ -15,6 +15,7 @@ Based on: Qwen3.5 backbone with custom text projection
|
||||
|
||||
Target models:
|
||||
- athrael-soju/colqwen3.5-4.5B-v3
|
||||
- vultr/VultronRetrieverPrime-Qwen3.5-8B
|
||||
"""
|
||||
|
||||
from collections.abc import Iterable, Mapping
|
||||
@@ -166,12 +167,19 @@ class ColQwen3_5Model(
|
||||
or 128 # default from reference implementation
|
||||
)
|
||||
|
||||
# ColPali defines `custom_text_proj = nn.Linear(hidden, dim)`, i.e.
|
||||
# bias=True by default, and the trained ColQwen3.5 checkpoints ship a
|
||||
# `custom_text_proj.bias`. Construct with a bias and zero-initialize it:
|
||||
# a (legacy) bias-less checkpoint then behaves identically to bias=False,
|
||||
# while load_weights() below picks up a trained bias instead of silently
|
||||
# dropping it (which shifts every per-token vector and the MaxSim ranking).
|
||||
self.custom_text_proj = nn.Linear(
|
||||
hidden_size,
|
||||
self.embed_dim,
|
||||
bias=False,
|
||||
bias=True,
|
||||
dtype=head_dtype,
|
||||
)
|
||||
nn.init.zeros_(self.custom_text_proj.bias)
|
||||
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
@@ -627,6 +627,20 @@ class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
|
||||
)
|
||||
|
||||
|
||||
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""ColQwen3.5 (late-interaction retrieval) inherits Qwen3.5's mamba cache
|
||||
handling and additionally serves BIDIRECTIONAL attention: ColPali-style
|
||||
document/query encoding attends over the whole sequence, not causally. Set
|
||||
is_causal=False so Qwen3NextAttention builds its full_attention layers with
|
||||
AttentionType.ENCODER_ONLY (the linear_attention GatedDeltaNet layers are
|
||||
unaffected). Generation arches keep the parent (causal) and are untouched.
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
model_config.hf_config.is_causal = False
|
||||
|
||||
|
||||
class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
@@ -656,7 +670,7 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = {
|
||||
"ColBERTJinaRobertaModel": JinaRobertaModelConfig,
|
||||
"ColQwen3_5": Qwen3_5ForConditionalGenerationConfig,
|
||||
"ColQwen3_5": ColQwen3_5Config,
|
||||
"DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig,
|
||||
"DeepseekV32ForCausalLM": DeepseekV32ForCausalLM,
|
||||
"DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501
|
||||
|
||||
@@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
from .interfaces import (
|
||||
EagleModelMixin,
|
||||
@@ -267,6 +268,15 @@ class Qwen3NextAttention(nn.Module):
|
||||
dual_chunk_attention_config=self.dual_chunk_attention_config,
|
||||
)
|
||||
|
||||
# Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL
|
||||
# attention on the full_attention layers; they set config.is_causal=False
|
||||
# via a VerifyAndUpdateConfig handler. Generation models leave is_causal
|
||||
# unset (-> causal/DECODER), so this is a no-op for them. Mirrors qwen3.py.
|
||||
attn_type = (
|
||||
AttentionType.DECODER
|
||||
if getattr(config, "is_causal", True)
|
||||
else AttentionType.ENCODER_ONLY
|
||||
)
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -275,6 +285,7 @@ class Qwen3NextAttention(nn.Module):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
attn_type=attn_type,
|
||||
**{
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": self.dual_chunk_attention_config,
|
||||
|
||||
Reference in New Issue
Block a user