[Model] Add Clef-Flash support (#61308)

Signed-off-by: Tianyu Guo <guoty@inferact.ai>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Tianyu Guo
2026-10-13 05:46:37 +00:00
committed by GitHub
co-authored by Codex Isotr0py
parent 0d2689e0df
commit 380638afbb
11 changed files with 580 additions and 361 deletions
+34
View File
@@ -70,6 +70,40 @@ def _load_groundtruth(filename: str) -> dict:
return json.load(f)
@pytest.mark.parametrize("has_joint_head", [False, True])
@pytest.mark.parametrize(
"architecture", ["Qwen3_5ForConditionalGeneration", "ColQwen3_5"]
)
def test_qwen35_joint_head_selects_clef_flash(tmp_path, has_joint_head, architecture):
"""Only a Qwen3.5 generation checkpoint with a joint head selects Clef."""
from transformers import Qwen3_5Config
head_config = {
"hidden_size": 4096,
"width": 1024,
"routing_layers": 2,
"layers": 4,
"heads": 16,
"feedforward": 4096,
}
if has_joint_head:
(tmp_path / "joint_head_config.json").write_text(json.dumps(head_config))
config = Qwen3_5Config(architectures=[architecture])
config.name_or_path = str(tmp_path)
model_config = SimpleNamespace(
hf_config=config,
hf_text_config=config.text_config,
_supports_multimodal_for_mm_prefix=lambda: True,
)
converted = ModelConfig.get_model_arch_config(model_config)
if has_joint_head and architecture == "Qwen3_5ForConditionalGeneration":
assert converted.architectures == ["ClefFlashForJointSchema"]
assert config.joint_head_config == head_config
else:
assert converted.architectures == [architecture]
assert not hasattr(config, "joint_head_config")
def _assert_model_arch_config(
model_config, expected: dict, check_head_size: bool = True
):
@@ -33,6 +33,7 @@ from vllm.entrypoints.generate.structured_decisions.serving import (
parse_questions,
)
from vllm.entrypoints.generate.structured_decisions.strategies import (
ClefFlashStrategy,
DiffusionGemmaCanvasStrategy,
JointSchemaStrategy,
LiquidStrategy,
@@ -94,6 +95,8 @@ def test_strategy_selection():
# Clef-Omni scores options with its head, so it reads no logprobs.
clef = model("ClefOmniForJointSchema", "raw_logits")
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
clef_flash = model("ClefFlashForJointSchema", "raw_logits")
assert select_read_strategy(clef_flash, ("token_classify",)) is ClefFlashStrategy
def test_route_is_registered_by_default():
@@ -366,8 +369,8 @@ JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
)
@pytest.fixture
def joint_schema(qwen, engine_and_renderer):
@pytest.fixture(params=[JointSchemaStrategy, ClefFlashStrategy])
def joint_schema(request, qwen, engine_and_renderer):
tokenizer, _ = qwen
engine, online_renderer = engine_and_renderer
# The probability the head gives each option, by option id.
@@ -382,9 +385,7 @@ def joint_schema(qwen, engine_and_renderer):
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
engine.encode.side_effect = encode
strategy = JointSchemaStrategy(
ReadContext(engine, online_renderer, None, "auto", {})
)
strategy = request.param(ReadContext(engine, online_renderer, None, "auto", {}))
return serving(engine, strategy), engine, tokenizer
@@ -468,7 +469,10 @@ async def test_media_needs_a_model_that_takes_it(serving_and_engine):
@pytest.mark.asyncio
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
@pytest.mark.parametrize("num_images", [1, 2])
async def test_joint_schema_reads_media_before_the_state(
joint_schema, monkeypatch, num_images
):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
strategy.__dict__["_connector"] = MediaConnector()
@@ -491,23 +495,30 @@ async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypat
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image] * num_images)
)
assert isinstance(response, StructuredDecisionResponse)
placeholder = tokenizer.encode(
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
)
if isinstance(strategy, ClefFlashStrategy):
media_text = "<|vision_start|><|image_pad|><|vision_end|>" * num_images + "\n"
else:
media_text = "<|vision_start|><|image_pad|><|vision_end|>\n" * num_images
placeholder = tokenizer.encode(media_text, add_special_tokens=False)
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE}
}
if isinstance(strategy, ClefFlashStrategy):
assert rendered["mm_processor_kwargs"] == {
"videos_kwargs": {"do_sample_frames": False}
}
else:
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE}
}
prompt = engine.encode.call_args.args[0]
assert prompt["type"] == "multimodal"
ids = prompt["prompt_token_ids"]
expanded = len(strategy.prefix) + len(placeholder) + 2
assert ids[:expanded].count(image_pad) == 3
expanded = len(strategy.prefix) + len(placeholder) + 2 * num_images
assert ids[:expanded].count(image_pad) == 3 * num_images
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
assert tokenizer.decode(ids[expanded:]) == state_and_questions
assert response.answers["team"]["choice"] == "billing"
@@ -515,7 +526,9 @@ async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypat
@pytest.mark.asyncio
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
async def test_joint_schema_video_audio_follows_model_modalities(
joint_schema, monkeypatch
):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
video = MediaWithBytes(
@@ -532,6 +545,7 @@ async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
return video
async def fetch_audio_async(self, url):
assert not isinstance(strategy, ClefFlashStrategy)
if url == "https://example.org/silent.mp4":
raise ValueError("No audio stream found.")
return track
@@ -552,11 +566,17 @@ async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE},
"use_audio_in_video": True,
}
if isinstance(strategy, ClefFlashStrategy):
assert rendered["multi_modal_data"] == {"video": [video]}
assert rendered["mm_processor_kwargs"] == {
"videos_kwargs": {"do_sample_frames": False}
}
else:
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE},
"use_audio_in_video": True,
}
# A video without a decodable soundtrack is watched silently.
response = await serving.create_decision(
@@ -566,7 +586,8 @@ async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video]}
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
if not isinstance(strategy, ClefFlashStrategy):
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
# Audio clips cannot be read with videos that have soundtracks.
response = await serving.create_decision(
@@ -1,6 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
"""The shared Clef pooler against a port of the checkpoint's joint schema head,
which loops over questions where the pooler batches them."""
import math
@@ -11,7 +11,10 @@ import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
from vllm.model_executor.models.clef_joint_schema import (
JointSchemaHead,
JointSchemaPooler,
)
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
+4
View File
@@ -802,6 +802,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Salesforce/blip2-opt-2.7b",
extras={"6b": "Salesforce/blip2-opt-6.7b"},
),
"ClefFlashForJointSchema": _HfExamplesInfo(
"Cloudflare/clef-flash",
max_model_len=4096,
),
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
"CohereLabs/command-a-vision-07-2025",
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},