[Bugfix] Respect declared attention contract for ColQwen3.5 retrievers (#49372)
Signed-off-by: Athrael Soju <athrael.soju@gmail.com>
This commit is contained in:
@@ -7,6 +7,8 @@ ColBERT-style late interaction scoring (MaxSim). It produces per-token
|
||||
embeddings for both text and image inputs.
|
||||
"""
|
||||
|
||||
from types import SimpleNamespace
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
|
||||
@@ -154,12 +156,14 @@ def test_colqwen3_5_relevance_ordering(
|
||||
_run_relevance_test(vllm_runner, model, dtype=dtype)
|
||||
|
||||
|
||||
def test_colqwen3_5_config_enables_bidirectional_attention() -> None:
|
||||
"""ColQwen3.5 retrieval must be served BIDIRECTIONAL (is_causal=False) so the
|
||||
full_attention layers build with AttentionType.ENCODER_ONLY. This guards the
|
||||
silent-causal regression (no GPU / model load needed)."""
|
||||
from types import SimpleNamespace
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("contract", "expected_is_causal"),
|
||||
[("causal", True), ("bidirectional", False)],
|
||||
)
|
||||
def test_colqwen3_5_config_applies_declared_attention_contract(
|
||||
contract: str,
|
||||
expected_is_causal: bool,
|
||||
) -> None:
|
||||
from vllm.model_executor.models.config import (
|
||||
MODELS_CONFIG_MAP,
|
||||
ColQwen3_5Config,
|
||||
@@ -167,6 +171,97 @@ def test_colqwen3_5_config_enables_bidirectional_attention() -> None:
|
||||
|
||||
assert MODELS_CONFIG_MAP["ColQwen3_5"] is ColQwen3_5Config
|
||||
|
||||
model_config = SimpleNamespace(hf_config=SimpleNamespace())
|
||||
hf_config = SimpleNamespace(retrieval_attention_contract=contract)
|
||||
text_config = SimpleNamespace()
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=text_config,
|
||||
)
|
||||
ColQwen3_5Config.verify_and_update_model_config(model_config)
|
||||
assert model_config.hf_config.is_causal is False
|
||||
assert hf_config.is_causal is expected_is_causal
|
||||
assert text_config.is_causal is expected_is_causal
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"hf_config",
|
||||
[
|
||||
SimpleNamespace(),
|
||||
SimpleNamespace(retrieval_attention_contract="unsupported"),
|
||||
SimpleNamespace(
|
||||
retrieval_attention_contract="causal",
|
||||
text_config=SimpleNamespace(retrieval_attention_contract="bidirectional"),
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_colqwen3_5_config_rejects_invalid_attention_contract(hf_config) -> None:
|
||||
from vllm.model_executor.models.config import ColQwen3_5Config
|
||||
|
||||
text_config = getattr(hf_config, "text_config", SimpleNamespace())
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=text_config,
|
||||
)
|
||||
with pytest.raises(ValueError, match="retrieval_attention_contract"):
|
||||
ColQwen3_5Config.verify_and_update_model_config(model_config)
|
||||
|
||||
|
||||
def test_colqwen3_5_bidirectional_contract_builds_encoder_only_attention(
|
||||
monkeypatch,
|
||||
) -> None:
|
||||
from vllm.model_executor.models import qwen3_next
|
||||
from vllm.model_executor.models.config import ColQwen3_5Config
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
hf_config = SimpleNamespace(retrieval_attention_contract="bidirectional")
|
||||
text_config = SimpleNamespace(
|
||||
hidden_size=256,
|
||||
num_attention_heads=2,
|
||||
num_key_value_heads=1,
|
||||
head_dim=128,
|
||||
max_position_embeddings=4096,
|
||||
rope_parameters={},
|
||||
rms_norm_eps=1e-6,
|
||||
)
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=hf_config,
|
||||
hf_text_config=text_config,
|
||||
)
|
||||
ColQwen3_5Config.verify_and_update_model_config(model_config)
|
||||
|
||||
captured = {}
|
||||
|
||||
class FakeAttention(torch.nn.Module):
|
||||
def __init__(self, *args, **kwargs) -> None:
|
||||
super().__init__()
|
||||
captured["attn_type"] = kwargs["attn_type"]
|
||||
|
||||
monkeypatch.setattr(qwen3_next, "get_tensor_model_parallel_world_size", lambda: 1)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next, "QKVParallelLinear", lambda *args, **kwargs: torch.nn.Identity()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next, "RowParallelLinear", lambda *args, **kwargs: torch.nn.Identity()
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next,
|
||||
"get_rope",
|
||||
lambda *args, **kwargs: SimpleNamespace(is_neox_style=False),
|
||||
)
|
||||
monkeypatch.setattr(
|
||||
qwen3_next, "Qwen3NextRMSNorm", lambda *args, **kwargs: torch.nn.Identity()
|
||||
)
|
||||
monkeypatch.setattr(qwen3_next, "Attention", FakeAttention)
|
||||
|
||||
qwen3_next.Qwen3NextAttention(text_config)
|
||||
|
||||
assert captured["attn_type"] is AttentionType.ENCODER_ONLY
|
||||
|
||||
|
||||
def test_colqwen3_5_encoder_only_attention_has_no_kv_cache_spec() -> None:
|
||||
from vllm.model_executor.layers.attention import Attention
|
||||
from vllm.v1.attention.backend import AttentionType
|
||||
|
||||
attention = SimpleNamespace(attn_type=AttentionType.ENCODER_ONLY)
|
||||
vllm_config = SimpleNamespace(cache_config=SimpleNamespace(block_size=16))
|
||||
|
||||
assert Attention.get_kv_cache_spec(attention, vllm_config) is None
|
||||
|
||||
@@ -769,17 +769,41 @@ class Qwen3_5ForConditionalGenerationConfig(VerifyAndUpdateConfig):
|
||||
|
||||
|
||||
class ColQwen3_5Config(Qwen3_5ForConditionalGenerationConfig):
|
||||
"""ColQwen3.5 (late-interaction retrieval) inherits Qwen3.5's mamba cache
|
||||
handling and additionally serves BIDIRECTIONAL attention: ColPali-style
|
||||
document/query encoding attends over the whole sequence, not causally. Set
|
||||
is_causal=False so Qwen3NextAttention builds its full_attention layers with
|
||||
AttentionType.ENCODER_ONLY (the linear_attention GatedDeltaNet layers are
|
||||
unaffected). Generation arches keep the parent (causal) and are untouched.
|
||||
"""
|
||||
"""Apply the attention contract declared by a ColQwen3.5 checkpoint."""
|
||||
|
||||
@staticmethod
|
||||
def verify_and_update_model_config(model_config: "ModelConfig") -> None:
|
||||
model_config.hf_config.is_causal = False
|
||||
configs = {
|
||||
id(config): config
|
||||
for config in (
|
||||
model_config.hf_config,
|
||||
model_config.hf_text_config,
|
||||
)
|
||||
}
|
||||
declarations = [
|
||||
contract
|
||||
for config in configs.values()
|
||||
if (contract := getattr(config, "retrieval_attention_contract", None))
|
||||
is not None
|
||||
]
|
||||
supported = {"causal", "bidirectional"}
|
||||
if not declarations:
|
||||
raise ValueError(
|
||||
"ColQwen3.5 checkpoints must declare "
|
||||
"retrieval_attention_contract as 'causal' or 'bidirectional'"
|
||||
)
|
||||
if (
|
||||
any(not isinstance(contract, str) for contract in declarations)
|
||||
or any(contract not in supported for contract in declarations)
|
||||
or len(set(declarations)) != 1
|
||||
):
|
||||
raise ValueError(
|
||||
"unsupported or conflicting ColQwen3.5 "
|
||||
f"retrieval_attention_contract declarations: {declarations!r}"
|
||||
)
|
||||
is_causal = declarations[0] == "causal"
|
||||
for config in configs.values():
|
||||
config.is_causal = is_causal
|
||||
|
||||
|
||||
class SnowflakeGteNewModelConfig(VerifyAndUpdateConfig):
|
||||
|
||||
Reference in New Issue
Block a user