[Model] Add Clef-Flash support (#61308)
Signed-off-by: Tianyu Guo <guoty@inferact.ai> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Codex
Isotr0py
parent
0d2689e0df
commit
380638afbb
@@ -70,6 +70,40 @@ def _load_groundtruth(filename: str) -> dict:
|
||||
return json.load(f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("has_joint_head", [False, True])
|
||||
@pytest.mark.parametrize(
|
||||
"architecture", ["Qwen3_5ForConditionalGeneration", "ColQwen3_5"]
|
||||
)
|
||||
def test_qwen35_joint_head_selects_clef_flash(tmp_path, has_joint_head, architecture):
|
||||
"""Only a Qwen3.5 generation checkpoint with a joint head selects Clef."""
|
||||
from transformers import Qwen3_5Config
|
||||
|
||||
head_config = {
|
||||
"hidden_size": 4096,
|
||||
"width": 1024,
|
||||
"routing_layers": 2,
|
||||
"layers": 4,
|
||||
"heads": 16,
|
||||
"feedforward": 4096,
|
||||
}
|
||||
if has_joint_head:
|
||||
(tmp_path / "joint_head_config.json").write_text(json.dumps(head_config))
|
||||
config = Qwen3_5Config(architectures=[architecture])
|
||||
config.name_or_path = str(tmp_path)
|
||||
model_config = SimpleNamespace(
|
||||
hf_config=config,
|
||||
hf_text_config=config.text_config,
|
||||
_supports_multimodal_for_mm_prefix=lambda: True,
|
||||
)
|
||||
converted = ModelConfig.get_model_arch_config(model_config)
|
||||
if has_joint_head and architecture == "Qwen3_5ForConditionalGeneration":
|
||||
assert converted.architectures == ["ClefFlashForJointSchema"]
|
||||
assert config.joint_head_config == head_config
|
||||
else:
|
||||
assert converted.architectures == [architecture]
|
||||
assert not hasattr(config, "joint_head_config")
|
||||
|
||||
|
||||
def _assert_model_arch_config(
|
||||
model_config, expected: dict, check_head_size: bool = True
|
||||
):
|
||||
|
||||
@@ -33,6 +33,7 @@ from vllm.entrypoints.generate.structured_decisions.serving import (
|
||||
parse_questions,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
ClefFlashStrategy,
|
||||
DiffusionGemmaCanvasStrategy,
|
||||
JointSchemaStrategy,
|
||||
LiquidStrategy,
|
||||
@@ -94,6 +95,8 @@ def test_strategy_selection():
|
||||
# Clef-Omni scores options with its head, so it reads no logprobs.
|
||||
clef = model("ClefOmniForJointSchema", "raw_logits")
|
||||
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
|
||||
clef_flash = model("ClefFlashForJointSchema", "raw_logits")
|
||||
assert select_read_strategy(clef_flash, ("token_classify",)) is ClefFlashStrategy
|
||||
|
||||
|
||||
def test_route_is_registered_by_default():
|
||||
@@ -366,8 +369,8 @@ JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def joint_schema(qwen, engine_and_renderer):
|
||||
@pytest.fixture(params=[JointSchemaStrategy, ClefFlashStrategy])
|
||||
def joint_schema(request, qwen, engine_and_renderer):
|
||||
tokenizer, _ = qwen
|
||||
engine, online_renderer = engine_and_renderer
|
||||
# The probability the head gives each option, by option id.
|
||||
@@ -382,9 +385,7 @@ def joint_schema(qwen, engine_and_renderer):
|
||||
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
|
||||
|
||||
engine.encode.side_effect = encode
|
||||
strategy = JointSchemaStrategy(
|
||||
ReadContext(engine, online_renderer, None, "auto", {})
|
||||
)
|
||||
strategy = request.param(ReadContext(engine, online_renderer, None, "auto", {}))
|
||||
return serving(engine, strategy), engine, tokenizer
|
||||
|
||||
|
||||
@@ -468,7 +469,10 @@ async def test_media_needs_a_model_that_takes_it(serving_and_engine):
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
|
||||
@pytest.mark.parametrize("num_images", [1, 2])
|
||||
async def test_joint_schema_reads_media_before_the_state(
|
||||
joint_schema, monkeypatch, num_images
|
||||
):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
strategy = serving.strategy
|
||||
strategy.__dict__["_connector"] = MediaConnector()
|
||||
@@ -491,23 +495,30 @@ async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypat
|
||||
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
|
||||
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image] * num_images)
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
|
||||
placeholder = tokenizer.encode(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
|
||||
)
|
||||
if isinstance(strategy, ClefFlashStrategy):
|
||||
media_text = "<|vision_start|><|image_pad|><|vision_end|>" * num_images + "\n"
|
||||
else:
|
||||
media_text = "<|vision_start|><|image_pad|><|vision_end|>\n" * num_images
|
||||
placeholder = tokenizer.encode(media_text, add_special_tokens=False)
|
||||
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
|
||||
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE}
|
||||
}
|
||||
if isinstance(strategy, ClefFlashStrategy):
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"videos_kwargs": {"do_sample_frames": False}
|
||||
}
|
||||
else:
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE}
|
||||
}
|
||||
prompt = engine.encode.call_args.args[0]
|
||||
assert prompt["type"] == "multimodal"
|
||||
ids = prompt["prompt_token_ids"]
|
||||
expanded = len(strategy.prefix) + len(placeholder) + 2
|
||||
assert ids[:expanded].count(image_pad) == 3
|
||||
expanded = len(strategy.prefix) + len(placeholder) + 2 * num_images
|
||||
assert ids[:expanded].count(image_pad) == 3 * num_images
|
||||
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
|
||||
assert tokenizer.decode(ids[expanded:]) == state_and_questions
|
||||
assert response.answers["team"]["choice"] == "billing"
|
||||
@@ -515,7 +526,9 @@ async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypat
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
|
||||
async def test_joint_schema_video_audio_follows_model_modalities(
|
||||
joint_schema, monkeypatch
|
||||
):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
strategy = serving.strategy
|
||||
video = MediaWithBytes(
|
||||
@@ -532,6 +545,7 @@ async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
|
||||
return video
|
||||
|
||||
async def fetch_audio_async(self, url):
|
||||
assert not isinstance(strategy, ClefFlashStrategy)
|
||||
if url == "https://example.org/silent.mp4":
|
||||
raise ValueError("No audio stream found.")
|
||||
return track
|
||||
@@ -552,11 +566,17 @@ async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
|
||||
)
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE},
|
||||
"use_audio_in_video": True,
|
||||
}
|
||||
if isinstance(strategy, ClefFlashStrategy):
|
||||
assert rendered["multi_modal_data"] == {"video": [video]}
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"videos_kwargs": {"do_sample_frames": False}
|
||||
}
|
||||
else:
|
||||
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE},
|
||||
"use_audio_in_video": True,
|
||||
}
|
||||
|
||||
# A video without a decodable soundtrack is watched silently.
|
||||
response = await serving.create_decision(
|
||||
@@ -566,7 +586,8 @@ async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert rendered["multi_modal_data"] == {"video": [video]}
|
||||
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
|
||||
if not isinstance(strategy, ClefFlashStrategy):
|
||||
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
|
||||
|
||||
# Audio clips cannot be read with videos that have soundtracks.
|
||||
response = await serving.create_decision(
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
|
||||
"""The shared Clef pooler against a port of the checkpoint's joint schema head,
|
||||
which loops over questions where the pooler batches them."""
|
||||
|
||||
import math
|
||||
@@ -11,7 +11,10 @@ import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
|
||||
from vllm.model_executor.models.clef_joint_schema import (
|
||||
JointSchemaHead,
|
||||
JointSchemaPooler,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
|
||||
@@ -802,6 +802,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Salesforce/blip2-opt-2.7b",
|
||||
extras={"6b": "Salesforce/blip2-opt-6.7b"},
|
||||
),
|
||||
"ClefFlashForJointSchema": _HfExamplesInfo(
|
||||
"Cloudflare/clef-flash",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Cohere2VisionForConditionalGeneration": _HfExamplesInfo(
|
||||
"CohereLabs/command-a-vision-07-2025",
|
||||
extras={"command-a-plus": "CohereLabs/command-a-plus-05-2026-bf16"},
|
||||
|
||||
Reference in New Issue
Block a user