[Model] Support Cloudflare/clef-omni on /v1/systemone (#61064)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Jiangyun Zhu
2026-10-12 21:27:34 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Isotr0py
parent 314ab8cffd
commit 092c35dd15
20 changed files with 1592 additions and 69 deletions
@@ -1,12 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from io import BytesIO
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import numpy as np
import pybase64 as base64
import pytest
import torch
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
from PIL import Image
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
@@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import (
)
from vllm.entrypoints.generate.structured_decisions.serving import (
ServingStructuredDecisions,
state_text,
parse_questions,
)
from vllm.entrypoints.generate.structured_decisions.strategies import (
DiffusionGemmaCanvasStrategy,
JointSchemaStrategy,
NextTokenStrategy,
ReadContext,
reply_label_ids,
select_read_strategy,
state_text,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.register import (
init_exception_handler,
)
from vllm.inputs import mm_input
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.multimodal.media import MediaConnector, MediaWithBytes
from vllm.outputs import (
CompletionOutput,
PoolingOutput,
PoolingRequestOutput,
RequestOutput,
)
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
def test_strategy_selection():
qwen = "Qwen3ForCausalLM"
assert select_read_strategy(model(qwen)) is NextTokenStrategy
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
qwen, generate = "Qwen3ForCausalLM", ("generate",)
assert select_read_strategy(model(qwen), generate) is NextTokenStrategy
assert (
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"))
select_read_strategy(model(qwen, "processed_logprobs"), generate)
is NextTokenStrategy
)
assert (
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate)
is DiffusionGemmaCanvasStrategy
)
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
select_read_strategy(model("LlamaForCausalLM"))
select_read_strategy(model("LlamaForCausalLM"), generate)
with pytest.raises(ValueError, match="not raw_logits"):
select_read_strategy(model(qwen, "raw_logits"))
select_read_strategy(model(qwen, "raw_logits"), generate)
# A generative architecture served as a pooling model has no read strategy.
with pytest.raises(ValueError, match="need the generate task"):
select_read_strategy(model(qwen), ("embed",))
# Clef-Omni scores options with its head, so it reads no logprobs.
clef = model("ClefOmniForJointSchema", "raw_logits")
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
def test_route_is_registered_by_default():
@@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen):
@pytest.fixture
def serving_and_engine(qwen):
def engine_and_renderer(qwen):
tokenizer, _ = qwen
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
renderer = HfRenderer(
@@ -148,6 +173,19 @@ def serving_and_engine(qwen):
chat_template=None,
chat_template_content_format="auto",
)
yield engine, online_renderer
renderer.shutdown()
def serving(engine, strategy) -> ServingStructuredDecisions:
name = engine.model_config.model
models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)])
return ServingStructuredDecisions(models, strategy)
@pytest.fixture
def serving_and_engine(engine_and_renderer):
engine, online_renderer = engine_and_renderer
async def generate(prompt, params, request_id, **kwargs):
label_ids = params.logprob_token_ids
@@ -171,11 +209,7 @@ def serving_and_engine(qwen):
engine.generate.side_effect = generate
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
models = OpenAIServingModels(
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
)
yield ServingStructuredDecisions(models, strategy), engine
renderer.shutdown()
return serving(engine, strategy), engine
@pytest.mark.asyncio
@@ -256,3 +290,286 @@ def test_canvas_read():
strategy._read_input(
{"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4]
)
JOINT_SCHEMA_PROMPT = """<|im_start|>system
Read the complete state and schema. Decide every field jointly. Each answer \
must be exactly one of that field's allowed options.<|im_end|>
<|im_start|>user
STATE:
{"customer":{"tier":"gold"},"ticket":"Charged twice"}
SCHEMA FIELDS:
FIELD 1
ID: team
TYPE: choice
INSTRUCTION: Which team?
ALLOWED OPTIONS:
OPTION 1: {"description":"payments","option_id":"billing"}
OPTION 2: {"option_id":"security"}
OPTION 3: {"description":"deliveries","option_id":"shipping"}
END FIELD
FIELD 2
ID: urgent
TYPE: noul
INSTRUCTION: urgent
ALLOWED OPTIONS:
OPTION 1: {"description":"money is moving","option_id":"true"}
OPTION 2: {"description":"The proposition is false or the answer is no.",\
"option_id":"false"}
END FIELD
FIELD 3
ID: tone
TYPE: score
INSTRUCTION: How upset?
ALLOWED OPTIONS:
OPTION 1: {"description":"calm","option_id":"0"}
OPTION 2: {"description":"annoyed","option_id":"1"}
OPTION 3: {"description":"furious","option_id":"2"}
END FIELD
<|im_end|>
<|im_start|>assistant
<think>
</think>
JOINT SCHEMA DECISIONS:"""
JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
state={"ticket": "Charged twice", "customer": {"tier": "gold"}},
questions={
"team": {
"type": "choice",
"instructions": "Which team?",
"criteria": {
"shipping": "deliveries",
"billing": "payments",
"security": None,
},
},
"urgent": {"type": "noul", "criteria": {"true": "money is moving"}},
"tone": {
"type": "score",
"instructions": "How upset?",
"criteria": ["calm", "annoyed", "furious"],
},
},
)
@pytest.fixture
def joint_schema(qwen, engine_and_renderer):
tokenizer, _ = qwen
engine, online_renderer = engine_and_renderer
# The probability the head gives each option, by option id.
scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2}
scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3}
async def encode(prompt, params, request_id, **kwargs):
ids = prompt["prompt_token_ids"]
spans = params.extra_kwargs["joint_schema"]["option_spans"]
options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans]
data = torch.tensor([scores[o["option_id"]] for o in options])
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
engine.encode.side_effect = encode
strategy = JointSchemaStrategy(
ReadContext(engine, online_renderer, None, "auto", {})
)
return serving(engine, strategy), engine, tokenizer
@pytest.mark.asyncio
async def test_joint_schema_reads_every_question_at_once(joint_schema):
serving, engine, tokenizer = joint_schema
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt")
)
assert isinstance(response, StructuredDecisionResponse)
engine.encode.assert_called_once()
engine.check_admission.assert_called_once_with(1)
prompt, params = engine.encode.call_args.args[:2]
ids = prompt["prompt_token_ids"]
assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT
assert prompt["cache_salt"] == "salt"
assert params.task == "token_classify"
schema = params.extra_kwargs["joint_schema"]
assert schema["question_types"] == [1, 0, 2]
assert schema["option_counts"] == [3, 2, 3]
assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [
"Which team?",
"urgent",
"How upset?",
]
# Options are read in the model's order and answered in the request's.
team = response.answers["team"]
assert team["choice"] == "billing"
assert list(team["probabilities"]) == ["shipping", "billing", "security"]
assert team["probabilities"] == pytest.approx(
{"shipping": 0.2, "billing": 0.7, "security": 0.1}
)
assert response.answers["urgent"]["noul"] == pytest.approx(0.9)
assert response.answers["tone"]["score"] == pytest.approx(1.1)
assert response.usage.input_tokens == len(ids)
assert response.usage.output_tokens == 0
assert all(d.label_mass == 1.0 for d in response.diagnostics.values())
@pytest.mark.asyncio
async def test_joint_schema_rejects_instructions(joint_schema):
serving, engine, _ = joint_schema
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.")
)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
engine.encode.assert_not_called()
def test_joint_schema_truncates_the_state(joint_schema):
serving, _, _ = joint_schema
strategy = serving.strategy
request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST)
questions = parse_questions(request, strategy.limits())
prompt, schema, _ = strategy.encode(questions, "word " * 100)
strategy.max_length = len(prompt) - 50
truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100)
# The state loses 50 tokens and the questions stay at the end.
assert len(truncated) == strategy.max_length
assert truncated[-200:] == prompt[-200:]
assert truncated_schema == schema
strategy.max_length = 100
with pytest.raises(StructuredDecisionError, match="questions take"):
strategy.encode(questions, "word")
@pytest.mark.asyncio
async def test_media_needs_a_model_that_takes_it(serving_and_engine):
serving, engine = serving_and_engine
request = StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"]
)
response = await serving.create_decision(request)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert "does not take image inputs" in response.error.message
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
strategy.__dict__["_connector"] = MediaConnector()
image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
rendered: dict[str, Any] = {}
async def process_for_engine_async(prompt, arrival_time):
rendered.update(prompt)
# The image's placeholder expands to three tokens.
ids = [
token
for pad in prompt["prompt_token_ids"]
for token in ([pad] * 3 if pad == image_pad else [pad])
]
return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt"))
renderer = strategy.context.online_renderer.renderer
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
buffer = BytesIO()
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
)
assert isinstance(response, StructuredDecisionResponse)
placeholder = tokenizer.encode(
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
)
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE}
}
prompt = engine.encode.call_args.args[0]
assert prompt["type"] == "multimodal"
ids = prompt["prompt_token_ids"]
expanded = len(strategy.prefix) + len(placeholder) + 2
assert ids[:expanded].count(image_pad) == 3
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
assert tokenizer.decode(ids[expanded:]) == state_and_questions
assert response.answers["team"]["choice"] == "billing"
assert response.usage.input_tokens == len(ids)
@pytest.mark.asyncio
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
video = MediaWithBytes(
(np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes"
)
track = (np.zeros(160, np.float32), 16000)
rendered: dict[str, Any] = {}
class FakeConnector:
async def fetch_image_async(self, url):
raise AssertionError("no images in this test")
async def load_from_url_async(self, url, media_io, *, fetch_timeout=None):
return video
async def fetch_audio_async(self, url):
if url == "https://example.org/silent.mp4":
raise ValueError("No audio stream found.")
return track
strategy.__dict__["_connector"] = FakeConnector()
async def process_for_engine_async(prompt, arrival_time):
rendered.clear()
rendered.update(prompt)
return mm_input(prompt["prompt_token_ids"], {}, {}, {})
renderer = strategy.context.online_renderer.renderer
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"]
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE},
"use_audio_in_video": True,
}
# A video without a decodable soundtrack is watched silently.
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"]
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video]}
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
# Audio clips cannot be read with videos that have soundtracks.
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST,
videos=["https://example.org/talking.mp4"],
audio=["https://example.org/clip.wav"],
)
)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
]
def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
"""A source with explicit allow patterns (e.g. a standalone head file in
the repo root) must not be validated against the backbone's index."""
save_file(
{"backbone.weight": torch.zeros(1)},
tmp_path / "model-00001-of-00001.safetensors",
)
save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
)
)
loader = DefaultModelLoader(LoadConfig())
secondary = DefaultModelLoader.Source(
model_or_path=str(tmp_path),
revision=None,
prefix="pooler.head.",
allow_patterns_overrides=["joint_head.safetensors"],
)
assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
"pooler.head.head.weight"
]
# The primary source still filters files the backbone index doesn't list.
primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
which loops over questions where the pooler batches them."""
import math
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
HIDDEN, VOCAB = 64, 500
TYPES = [0, 1, 1, 2]
COUNTS = [2, 3, 1, 5]
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
for one record, with ``spans`` the option spans of each question."""
hidden = head.hidden_norm(hidden)
memory = head.memory_projection(hidden).unsqueeze(0)
global_vector = hidden[-1]
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
lexicals = [
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
]
options = torch.cat(
[
head.option_context_projection(c)
+ head.option_lexical_projection(lex)
+ head.option_question_projection(questions[i]).unsqueeze(0)
for i, (c, lex) in enumerate(zip(contexts, lexicals))
]
).unsqueeze(0)
for layer in head.evidence_layers:
options = layer(options, memory)
routed = torch.split(options[0], [len(q) for q in spans])
base = head.question_projection(questions)
summaries = []
for field, opts in zip(base, routed):
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
summaries.append((weights[:, None] * opts).sum(0))
fields = (
base
+ head.option_summary_norm(torch.stack(summaries))
+ head.global_projection(global_vector)
+ head.type_embedding(torch.tensor(TYPES))
).unsqueeze(0)
for layer in head.layers:
fields = layer(fields, memory)
fields = head.field_norm(fields[0])
logits = []
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
anchor = F.normalize(questions[i] + global_vector, dim=-1)
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
opts = head.option_norm(opts)
field = field.expand_as(opts)
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
joint = joint + head.residual_scorer(features).squeeze(-1)
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
return logits
@pytest.fixture
def pooler():
torch.manual_seed(0)
head = JointSchemaHead(
hidden_size=HIDDEN,
width=32,
routing_layers=2,
layers=2,
heads=4,
feedforward=48,
).eval()
with torch.no_grad():
for p in head.parameters():
p.add_(torch.randn_like(p) * 0.2)
embedding = torch.randn(VOCAB, HIDDEN)
config = SimpleNamespace(
scheduler_config=SimpleNamespace(
enable_chunked_prefill=False, async_scheduling=False
)
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=config,
):
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
return pooler, embedding
def schema(length: int):
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
pos = 20
question_spans: list[tuple[int, int]] = []
spans: list[list[tuple[int, int]]] = []
for count in COUNTS:
question_spans.append((pos, pos + 4))
pos += 6
spans.append([])
for k in range(count):
n = 1 + (k * 3) % 4
spans[-1].append((pos, pos + n))
pos += n + 1
assert pos < length
extra = {
"question_types": TYPES,
"question_spans": [[s - length, e - length] for s, e in question_spans],
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
"option_counts": COUNTS,
}
return {"joint_schema": extra}, question_spans, spans
@torch.inference_mode()
def test_pooler_matches_reference(pooler):
pooler, embedding = pooler
lengths = [120, 90, 100]
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
hidden = torch.randn(sum(lengths), HIDDEN)
first, first_questions, first_spans = schema(lengths[0])
last, last_questions, last_spans = schema(lengths[2])
params = [
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
# A prompt without a schema, like the profiling run's.
PoolingParams(task="token_classify", use_activation=True),
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
]
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
for i, ids in enumerate(token_ids):
padded[i, : len(ids)] = ids
metadata = PoolingMetadata(
prompt_lens=torch.tensor(lengths),
prompt_token_ids=None,
prompt_token_ids_cpu=padded,
pooling_params=params,
pooling_states=[PoolingStates() for _ in lengths],
)
metadata.build_pooling_cursor(
torch.tensor(lengths).numpy(),
seq_lens_cpu=torch.tensor(lengths),
device=torch.device("cpu"),
)
probs, empty, logits = pooler(hidden, metadata)
first_hidden, _, last_hidden = torch.split(hidden, lengths)
expected = reference_logits(
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
)
expected_probs = torch.cat([q.softmax(-1) for q in expected])
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
assert empty.numel() == 0
expected = reference_logits(
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
)
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
f"Got short={short_tokens}, long={long_tokens}"
)
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
"""Audio and video tokens interleave by the fps the video was processed
at, as in HF, not by the default of one frame a second."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
rng = np.random.RandomState(42)
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
audio = rng.rand(32000).astype(np.float32)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
mm_kwargs: dict[str, Any] = {
"use_audio_in_video": True,
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [video], "audio": [(audio, 16000)]}
),
hf_processor_mm_kwargs=mm_kwargs,
)
expected = processor.info.get_hf_processor()(
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
)
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
+4
View File
@@ -1351,6 +1351,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
),
"ClefOmniForJointSchema": _HfExamplesInfo(
"Cloudflare/clef-omni",
max_model_len=4096,
),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
+16
View File
@@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
ClefOmniVideoBackend,
DynamicVideoBackend,
Glm5NextVideoBackend,
GLM46VVideoBackend,
@@ -1813,3 +1814,18 @@ def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# One initial seek, then pure walking -- no re-seek churn.
assert cap.seeks == 1
def test_clef_omni_backend_samples_reference_frames(tmp_path):
"""Clef-Omni samples the first frame at or after each 0.5 s boundary
(the reference's 2 fps timestamp sampling), not uniform strides."""
pytest.importorskip("cv2")
# 2 s at 30 fps: the reference reads frames 0, 15, 30 and 45, where
# uniform sampling would read 0, 19, 39 and 59.
path = _write_gray_video(tmp_path, total_frames=60, fps=30)
frames, metadata = ClefOmniVideoBackend.load_bytes(path.read_bytes())
assert metadata["frames_indices"] == [0, 15, 30, 45]
assert metadata["do_sample_frames"] is False
for frame, idx in zip(frames, [0, 15, 30, 45]):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1