From 3c8e49596c3fd34ae82c8c5d881e91a38663639b Mon Sep 17 00:00:00 2001 From: Athrael Soju Date: Mon, 22 Jun 2026 10:25:54 +0100 Subject: [PATCH] [Model] ColQwen3.5: fix retrieval correctness (bias + bidirectional) (#46108) Signed-off-by: Athrael Soju Co-authored-by: Claude Opus 4.8 (1M context) --- docs/models/pooling_models/token_embed.md | 2 +- .../pooling/score/colqwen3_5_rerank_online.py | 18 +++++++++++++++++- .../multimodal/pooling/test_colqwen3_5.py | 18 ++++++++++++++++++ .../layers/attention/attention.py | 9 ++++++++- vllm/model_executor/models/colqwen3_5.py | 10 +++++++++- vllm/model_executor/models/config.py | 16 +++++++++++++++- vllm/model_executor/models/qwen3_next.py | 11 +++++++++++ 7 files changed, 79 insertions(+), 5 deletions(-) diff --git a/docs/models/pooling_models/token_embed.md b/docs/models/pooling_models/token_embed.md index 02050b7f50..0c2a322e80 100644 --- a/docs/models/pooling_models/token_embed.md +++ b/docs/models/pooling_models/token_embed.md @@ -61,7 +61,7 @@ Models of any architecture can be converted into embedding models using `--conve | `ColModernVBertForRetrieval` | ColModernVBERT | T / I | `ModernVBERT/colmodernvbert-merged` | | | | `ColPaliForRetrieval` | ColPali | T / I | `vidore/colpali-v1.3-hf` | | | | `ColQwen3` | Qwen3-VL | T / I | `TomoroAI/tomoro-colqwen3-embed-4b`, `TomoroAI/tomoro-colqwen3-embed-8b` | | | -| `ColQwen3_5` | ColQwen3.5 | T + I + V | `athrael-soju/colqwen3.5-4.5B-v3` | | | +| `ColQwen3_5` | ColQwen3.5 | T + I + V | `athrael-soju/colqwen3.5-4.5B-v3`, `vultr/VultronRetrieverPrime-Qwen3.5-8B` | | | | `OpsColQwen3Model` | Qwen3-VL | T / I | `OpenSearch-AI/Ops-Colqwen3-4B`, `OpenSearch-AI/Ops-Colqwen3-8B` | | | | `Qwen3VLNemotronEmbedModel` | Qwen3-VL | T / I | `nvidia/nemotron-colembed-vl-4b-v2`, `nvidia/nemotron-colembed-vl-8b-v2` | ✅︎ | ✅︎ | | `*ForConditionalGeneration`C, `*ForCausalLM`C, etc. | Generative models | \* | N/A | \* | \* | diff --git a/examples/pooling/score/colqwen3_5_rerank_online.py b/examples/pooling/score/colqwen3_5_rerank_online.py index c64bcfc81f..00746634d5 100644 --- a/examples/pooling/score/colqwen3_5_rerank_online.py +++ b/examples/pooling/score/colqwen3_5_rerank_online.py @@ -7,11 +7,27 @@ ColQwen3.5 is a multi-modal ColBERT-style model based on Qwen3.5. It produces per-token embeddings and uses MaxSim scoring for retrieval and reranking. Supports both text and image inputs. +Works for any ColQwen3.5 checkpoint, e.g. `athrael-soju/colqwen3.5-4.5B-v3` +or `vultr/VultronRetrieverPrime-Qwen3.5-8B`. + Start the server with: - vllm serve athrael-soju/colqwen3.5-4.5B --max-model-len 4096 + vllm serve athrael-soju/colqwen3.5-4.5B-v3 --max-model-len 4096 \ + --mm-processor-kwargs '{"min_pixels": 65536, "max_pixels": 1835008}' Then run this script: python colqwen3_5_rerank_online.py + +Parity note (matching the native colpali ColQwen3_5Processor pipeline): + - Visual-token budget: ColQwen3_5Processor uses max_num_visual_tokens=1792, + i.e. max_pixels = 1792 * (patch_size*merge_size)^2 = 1792 * 32^2 = 1835008 + (with min_pixels = shortest_edge = 65536). Pass these via --mm-processor-kwargs + as above; the default budget gives fewer visual tokens and lower retrieval ndcg. + - When you build prompts yourself (token_embed), reproduce the processor exactly: + image (document): wrap in the instruction template + "<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>" + "Describe the image.<|im_end|><|endoftext|>" + query: append the augmentation suffix + "<|endoftext|>" * 10 + Omitting these reproduces a silent ~2.5 ndcg@10 drop vs the native pipeline. """ import requests diff --git a/tests/models/multimodal/pooling/test_colqwen3_5.py b/tests/models/multimodal/pooling/test_colqwen3_5.py index 2b6a5a263c..3513bd025b 100644 --- a/tests/models/multimodal/pooling/test_colqwen3_5.py +++ b/tests/models/multimodal/pooling/test_colqwen3_5.py @@ -152,3 +152,21 @@ def test_colqwen3_5_relevance_ordering( dtype: str, ) -> None: _run_relevance_test(vllm_runner, model, dtype=dtype) + + +def test_colqwen3_5_config_enables_bidirectional_attention() -> None: + """ColQwen3.5 retrieval must be served BIDIRECTIONAL (is_causal=False) so the + full_attention layers build with AttentionType.ENCODER_ONLY. This guards the + silent-causal regression (no GPU / model load needed).""" + from types import SimpleNamespace + + from vllm.model_executor.models.config import ( + MODELS_CONFIG_MAP, + ColQwen3_5Config, + ) + + assert MODELS_CONFIG_MAP["ColQwen3_5"] is ColQwen3_5Config + + model_config = SimpleNamespace(hf_config=SimpleNamespace()) + ColQwen3_5Config.verify_and_update_model_config(model_config) + assert model_config.hf_config.is_causal is False diff --git a/vllm/model_executor/layers/attention/attention.py b/vllm/model_executor/layers/attention/attention.py index cdfe9fa1bc..3eec58aafa 100644 --- a/vllm/model_executor/layers/attention/attention.py +++ b/vllm/model_executor/layers/attention/attention.py @@ -581,7 +581,14 @@ class Attention(nn.Module, AttentionLayerBase): def get_kv_cache_spec(self, vllm_config: VllmConfig) -> KVCacheSpec | None: # Block size may get updated after model loading, refresh it block_size = vllm_config.cache_config.block_size - # Should not be called for enc-dec or encoder-only attention. + # Encoder-only attention is prefill-only and keeps no autoregressive KV + # cache. In hybrid models (e.g. Qwen3.5 / ColQwen3.5: GatedDeltaNet + # linear_attention interleaved with full_attention) the runner iterates + # every attention module to build the KV-cache spec, so an ENCODER_ONLY + # full_attention layer reaches here; it contributes no KV cache group. + if self.attn_type in (AttentionType.ENCODER_ONLY, AttentionType.ENCODER): + return None + # Should not be called for enc-dec attention. assert self.attn_type == AttentionType.DECODER quant_mode = get_kv_quant_mode(self.kv_cache_dtype) if self.sliding_window is not None: diff --git a/vllm/model_executor/models/colqwen3_5.py b/vllm/model_executor/models/colqwen3_5.py index 5c28fb6d37..1b481ea2f4 100644 --- a/vllm/model_executor/models/colqwen3_5.py +++ b/vllm/model_executor/models/colqwen3_5.py @@ -15,6 +15,7 @@ Based on: Qwen3.5 backbone with custom text projection Target models: - athrael-soju/colqwen3.5-4.5B-v3 +- vultr/VultronRetrieverPrime-Qwen3.5-8B """ from collections.abc import Iterable, Mapping @@ -166,12 +167,19 @@ class ColQwen3_5Model( or 128 # default from reference implementation ) + # ColPali defines `custom_text_proj = nn.Linear(hidden, dim)`, i.e. + # bias=True by default, and the trained ColQwen3.5 checkpoints ship a + # `custom_text_proj.bias`. Construct with a bias and zero-initialize it: + # a (legacy) bias-less checkpoint then behaves identically to bias=False, + # while load_weights() below picks up a trained bias instead of silently + # dropping it (which shifts every per-token vector and the MaxSim ranking). self.custom_text_proj = nn.Linear( hidden_size, self.embed_dim, - bias=False, + bias=True, dtype=head_dtype, ) + nn.init.zeros_(self.custom_text_proj.bias) pooler_config = vllm_config.model_config.pooler_config assert pooler_config is not None diff --git a/vllm/model_executor/models/config.py b/vllm/model_executor/models/config.py index 6b21ef8308..ac67614986 100644 --- a/vllm/model_executor/models/config.py +++ b/vllm/model_executor/models/config.py @@ -627,6 +627,20 @@ class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig): ) +class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig): + """ColQwen3.5 (late-interaction retrieval) inherits Qwen3.5's mamba cache + handling and additionally serves BIDIRECTIONAL attention: ColPali-style + document/query encoding attends over the whole sequence, not causally. Set + is_causal=False so Qwen3NextAttention builds its full_attention layers with + AttentionType.ENCODER_ONLY (the linear_attention GatedDeltaNet layers are + unaffected). Generation arches keep the parent (causal) and are untouched. + """ + + @staticmethod + def verify_and_update_model_config(model_config: "ModelConfig") -> None: + model_config.hf_config.is_causal = False + + class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig): @staticmethod def verify_and_update_model_config(model_config: "ModelConfig") -> None: @@ -656,7 +670,7 @@ class VoyageQwen3BidirectionalEmbedModelConfig(VerifyAndUpdateConfig): MODELS_CONFIG_MAP: dict[str, type[VerifyAndUpdateConfig]] = { "ColBERTJinaRobertaModel": JinaRobertaModelConfig, - "ColQwen3_5": Qwen3_5ForConditionalGenerationConfig, + "ColQwen3_5": ColQwen3_5Config, "DeepseekV4ForCausalLM": DeepseekV4ForCausalLMConfig, "DeepseekV32ForCausalLM": DeepseekV32ForCausalLM, "DiffusionGemmaForBlockDiffusion": DiffusionGemmaModelForBlockDiffusionConfig, # noqa: E501 diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index 2ab08290fb..acce2a7679 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -62,6 +62,7 @@ from vllm.model_executor.models.utils import sequence_parallel_chunk from vllm.platforms import current_platform from vllm.sequence import IntermediateTensors from vllm.transformers_utils.configs.qwen3_next import Qwen3NextConfig +from vllm.v1.attention.backend import AttentionType from .interfaces import ( EagleModelMixin, @@ -267,6 +268,15 @@ class Qwen3NextAttention(nn.Module): dual_chunk_attention_config=self.dual_chunk_attention_config, ) + # Late-interaction retrieval models (e.g. ColQwen3.5) run BIDIRECTIONAL + # attention on the full_attention layers; they set config.is_causal=False + # via a VerifyAndUpdateConfig handler. Generation models leave is_causal + # unset (-> causal/DECODER), so this is a no-op for them. Mirrors qwen3.py. + attn_type = ( + AttentionType.DECODER + if getattr(config, "is_causal", True) + else AttentionType.ENCODER_ONLY + ) self.attn = Attention( self.num_heads, self.head_dim, @@ -275,6 +285,7 @@ class Qwen3NextAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", + attn_type=attn_type, **{ "layer_idx": extract_layer_index(prefix), "dual_chunk_attention_config": self.dual_chunk_attention_config,