diff --git a/docs/serving/online_serving/structured_decisions.md b/docs/serving/online_serving/structured_decisions.md
index a9ed5a2ee5..4d8d65d89e 100644
--- a/docs/serving/online_serving/structured_decisions.md
+++ b/docs/serving/online_serving/structured_decisions.md
@@ -130,6 +130,32 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a
| options per `choice` | 26 |
| levels per `score` | 10 |
+## Clef-Omni
+
+[Clef-Omni](https://huggingface.co/Cloudflare/clef-omni) is a decision model:
+a joint schema head on the Qwen3-Omni thinker scores every option of every
+question in one forward pass. vLLM serves it as a pooling model.
+
+```bash
+vllm serve Cloudflare/clef-omni
+```
+
+- A request is one read. The prompt is the one the checkpoint's
+ `joint_schema_model.py` builds: the state, then every question with its
+ options. A JSON state is written as compact JSON with sorted keys.
+- A state too long for the model is truncated to fit, as the reference does.
+ A request whose questions alone exceed 64,000 tokens, or `max_model_len`,
+ returns 400.
+- `instructions` returns 400, since the prompt is fixed. `chat_template_kwargs`
+ has no effect.
+- There are no labels, so `label_mass` is 1 and `confidence` is the chosen
+ option's probability. `usage.input_tokens` counts the prompt once, and
+ `output_tokens` is 0.
+- `images`, `audio` and `videos` are read as the reference does: placed
+ before the state, videos sampled at 2 fps, and a video's soundtrack heard
+ with it when every video has one. Audio clips cannot be sent together with
+ videos that have soundtracks.
+
## Request fields
| field | meaning |
@@ -139,6 +165,7 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a
| `instructions` | optional context placed ahead of the questions |
| `chat_template_kwargs` | passed to the chat template, for example `{"enable_thinking": false}` |
| `cache_salt` | optional [prefix-cache salt](../../usage/security.md#cache-salting) applied to every question |
+| `images`, `audio`, `videos` | optional media URLs or data URLs, for models that take them ([Clef-Omni](#clef-omni)) |
A question with any field other than `type`, `instructions` and `criteria` is
rejected with a 400.
diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py
index 41b9a8f012..8d38e61e62 100644
--- a/tests/entrypoints/generate/structured_decisions/test_strategies.py
+++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py
@@ -1,12 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import json
+from io import BytesIO
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
+import numpy as np
+import pybase64 as base64
import pytest
+import torch
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
+from PIL import Image
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
@@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import (
)
from vllm.entrypoints.generate.structured_decisions.serving import (
ServingStructuredDecisions,
- state_text,
+ parse_questions,
)
from vllm.entrypoints.generate.structured_decisions.strategies import (
DiffusionGemmaCanvasStrategy,
+ JointSchemaStrategy,
NextTokenStrategy,
ReadContext,
reply_label_ids,
select_read_strategy,
+ state_text,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
+from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.register import (
init_exception_handler,
)
+from vllm.inputs import mm_input
from vllm.logprobs import Logprob
-from vllm.outputs import CompletionOutput, RequestOutput
+from vllm.multimodal.media import MediaConnector, MediaWithBytes
+from vllm.outputs import (
+ CompletionOutput,
+ PoolingOutput,
+ PoolingRequestOutput,
+ RequestOutput,
+)
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
def test_strategy_selection():
- qwen = "Qwen3ForCausalLM"
- assert select_read_strategy(model(qwen)) is NextTokenStrategy
- assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
+ qwen, generate = "Qwen3ForCausalLM", ("generate",)
+ assert select_read_strategy(model(qwen), generate) is NextTokenStrategy
assert (
- select_read_strategy(model("DiffusionGemmaForBlockDiffusion"))
+ select_read_strategy(model(qwen, "processed_logprobs"), generate)
+ is NextTokenStrategy
+ )
+ assert (
+ select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate)
is DiffusionGemmaCanvasStrategy
)
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
- select_read_strategy(model("LlamaForCausalLM"))
+ select_read_strategy(model("LlamaForCausalLM"), generate)
with pytest.raises(ValueError, match="not raw_logits"):
- select_read_strategy(model(qwen, "raw_logits"))
+ select_read_strategy(model(qwen, "raw_logits"), generate)
+ # A generative architecture served as a pooling model has no read strategy.
+ with pytest.raises(ValueError, match="need the generate task"):
+ select_read_strategy(model(qwen), ("embed",))
+ # Clef-Omni scores options with its head, so it reads no logprobs.
+ clef = model("ClefOmniForJointSchema", "raw_logits")
+ assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
def test_route_is_registered_by_default():
@@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen):
@pytest.fixture
-def serving_and_engine(qwen):
+def engine_and_renderer(qwen):
tokenizer, _ = qwen
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
renderer = HfRenderer(
@@ -148,6 +173,19 @@ def serving_and_engine(qwen):
chat_template=None,
chat_template_content_format="auto",
)
+ yield engine, online_renderer
+ renderer.shutdown()
+
+
+def serving(engine, strategy) -> ServingStructuredDecisions:
+ name = engine.model_config.model
+ models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)])
+ return ServingStructuredDecisions(models, strategy)
+
+
+@pytest.fixture
+def serving_and_engine(engine_and_renderer):
+ engine, online_renderer = engine_and_renderer
async def generate(prompt, params, request_id, **kwargs):
label_ids = params.logprob_token_ids
@@ -171,11 +209,7 @@ def serving_and_engine(qwen):
engine.generate.side_effect = generate
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
- models = OpenAIServingModels(
- engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
- )
- yield ServingStructuredDecisions(models, strategy), engine
- renderer.shutdown()
+ return serving(engine, strategy), engine
@pytest.mark.asyncio
@@ -256,3 +290,286 @@ def test_canvas_read():
strategy._read_input(
{"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4]
)
+
+
+JOINT_SCHEMA_PROMPT = """<|im_start|>system
+Read the complete state and schema. Decide every field jointly. Each answer \
+must be exactly one of that field's allowed options.<|im_end|>
+<|im_start|>user
+STATE:
+{"customer":{"tier":"gold"},"ticket":"Charged twice"}
+
+SCHEMA FIELDS:
+
+FIELD 1
+ID: team
+TYPE: choice
+INSTRUCTION: Which team?
+ALLOWED OPTIONS:
+OPTION 1: {"description":"payments","option_id":"billing"}
+OPTION 2: {"option_id":"security"}
+OPTION 3: {"description":"deliveries","option_id":"shipping"}
+END FIELD
+
+FIELD 2
+ID: urgent
+TYPE: noul
+INSTRUCTION: urgent
+ALLOWED OPTIONS:
+OPTION 1: {"description":"money is moving","option_id":"true"}
+OPTION 2: {"description":"The proposition is false or the answer is no.",\
+"option_id":"false"}
+END FIELD
+
+FIELD 3
+ID: tone
+TYPE: score
+INSTRUCTION: How upset?
+ALLOWED OPTIONS:
+OPTION 1: {"description":"calm","option_id":"0"}
+OPTION 2: {"description":"annoyed","option_id":"1"}
+OPTION 3: {"description":"furious","option_id":"2"}
+END FIELD
+
+<|im_end|>
+<|im_start|>assistant
+
+
+
+
+JOINT SCHEMA DECISIONS:"""
+
+JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
+ state={"ticket": "Charged twice", "customer": {"tier": "gold"}},
+ questions={
+ "team": {
+ "type": "choice",
+ "instructions": "Which team?",
+ "criteria": {
+ "shipping": "deliveries",
+ "billing": "payments",
+ "security": None,
+ },
+ },
+ "urgent": {"type": "noul", "criteria": {"true": "money is moving"}},
+ "tone": {
+ "type": "score",
+ "instructions": "How upset?",
+ "criteria": ["calm", "annoyed", "furious"],
+ },
+ },
+)
+
+
+@pytest.fixture
+def joint_schema(qwen, engine_and_renderer):
+ tokenizer, _ = qwen
+ engine, online_renderer = engine_and_renderer
+ # The probability the head gives each option, by option id.
+ scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2}
+ scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3}
+
+ async def encode(prompt, params, request_id, **kwargs):
+ ids = prompt["prompt_token_ids"]
+ spans = params.extra_kwargs["joint_schema"]["option_spans"]
+ options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans]
+ data = torch.tensor([scores[o["option_id"]] for o in options])
+ yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
+
+ engine.encode.side_effect = encode
+ strategy = JointSchemaStrategy(
+ ReadContext(engine, online_renderer, None, "auto", {})
+ )
+ return serving(engine, strategy), engine, tokenizer
+
+
+@pytest.mark.asyncio
+async def test_joint_schema_reads_every_question_at_once(joint_schema):
+ serving, engine, tokenizer = joint_schema
+ response = await serving.create_decision(
+ StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt")
+ )
+ assert isinstance(response, StructuredDecisionResponse)
+
+ engine.encode.assert_called_once()
+ engine.check_admission.assert_called_once_with(1)
+ prompt, params = engine.encode.call_args.args[:2]
+ ids = prompt["prompt_token_ids"]
+ assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT
+ assert prompt["cache_salt"] == "salt"
+ assert params.task == "token_classify"
+ schema = params.extra_kwargs["joint_schema"]
+ assert schema["question_types"] == [1, 0, 2]
+ assert schema["option_counts"] == [3, 2, 3]
+ assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [
+ "Which team?",
+ "urgent",
+ "How upset?",
+ ]
+
+ # Options are read in the model's order and answered in the request's.
+ team = response.answers["team"]
+ assert team["choice"] == "billing"
+ assert list(team["probabilities"]) == ["shipping", "billing", "security"]
+ assert team["probabilities"] == pytest.approx(
+ {"shipping": 0.2, "billing": 0.7, "security": 0.1}
+ )
+ assert response.answers["urgent"]["noul"] == pytest.approx(0.9)
+ assert response.answers["tone"]["score"] == pytest.approx(1.1)
+ assert response.usage.input_tokens == len(ids)
+ assert response.usage.output_tokens == 0
+ assert all(d.label_mass == 1.0 for d in response.diagnostics.values())
+
+
+@pytest.mark.asyncio
+async def test_joint_schema_rejects_instructions(joint_schema):
+ serving, engine, _ = joint_schema
+ response = await serving.create_decision(
+ StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.")
+ )
+ assert isinstance(response, ErrorResponse)
+ assert response.error.code == 400
+ engine.encode.assert_not_called()
+
+
+def test_joint_schema_truncates_the_state(joint_schema):
+ serving, _, _ = joint_schema
+ strategy = serving.strategy
+ request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST)
+ questions = parse_questions(request, strategy.limits())
+ prompt, schema, _ = strategy.encode(questions, "word " * 100)
+ strategy.max_length = len(prompt) - 50
+ truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100)
+ # The state loses 50 tokens and the questions stay at the end.
+ assert len(truncated) == strategy.max_length
+ assert truncated[-200:] == prompt[-200:]
+ assert truncated_schema == schema
+ strategy.max_length = 100
+ with pytest.raises(StructuredDecisionError, match="questions take"):
+ strategy.encode(questions, "word")
+
+
+@pytest.mark.asyncio
+async def test_media_needs_a_model_that_takes_it(serving_and_engine):
+ serving, engine = serving_and_engine
+ request = StructuredDecisionRequest(
+ **JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"]
+ )
+ response = await serving.create_decision(request)
+ assert isinstance(response, ErrorResponse)
+ assert response.error.code == 400
+ assert "does not take image inputs" in response.error.message
+ engine.generate.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
+ serving, engine, tokenizer = joint_schema
+ strategy = serving.strategy
+ strategy.__dict__["_connector"] = MediaConnector()
+ image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
+ rendered: dict[str, Any] = {}
+
+ async def process_for_engine_async(prompt, arrival_time):
+ rendered.update(prompt)
+ # The image's placeholder expands to three tokens.
+ ids = [
+ token
+ for pad in prompt["prompt_token_ids"]
+ for token in ([pad] * 3 if pad == image_pad else [pad])
+ ]
+ return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt"))
+
+ renderer = strategy.context.online_renderer.renderer
+ monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
+ buffer = BytesIO()
+ Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
+ image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
+ response = await serving.create_decision(
+ StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
+ )
+ assert isinstance(response, StructuredDecisionResponse)
+
+ placeholder = tokenizer.encode(
+ "<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
+ )
+ assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
+ assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
+ assert rendered["mm_processor_kwargs"] == {
+ "images_kwargs": {"size": strategy.IMAGE_SIZE}
+ }
+ prompt = engine.encode.call_args.args[0]
+ assert prompt["type"] == "multimodal"
+ ids = prompt["prompt_token_ids"]
+ expanded = len(strategy.prefix) + len(placeholder) + 2
+ assert ids[:expanded].count(image_pad) == 3
+ state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
+ assert tokenizer.decode(ids[expanded:]) == state_and_questions
+ assert response.answers["team"]["choice"] == "billing"
+ assert response.usage.input_tokens == len(ids)
+
+
+@pytest.mark.asyncio
+async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
+ serving, engine, tokenizer = joint_schema
+ strategy = serving.strategy
+ video = MediaWithBytes(
+ (np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes"
+ )
+ track = (np.zeros(160, np.float32), 16000)
+ rendered: dict[str, Any] = {}
+
+ class FakeConnector:
+ async def fetch_image_async(self, url):
+ raise AssertionError("no images in this test")
+
+ async def load_from_url_async(self, url, media_io, *, fetch_timeout=None):
+ return video
+
+ async def fetch_audio_async(self, url):
+ if url == "https://example.org/silent.mp4":
+ raise ValueError("No audio stream found.")
+ return track
+
+ strategy.__dict__["_connector"] = FakeConnector()
+
+ async def process_for_engine_async(prompt, arrival_time):
+ rendered.clear()
+ rendered.update(prompt)
+ return mm_input(prompt["prompt_token_ids"], {}, {}, {})
+
+ renderer = strategy.context.online_renderer.renderer
+ monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
+
+ response = await serving.create_decision(
+ StructuredDecisionRequest(
+ **JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"]
+ )
+ )
+ assert isinstance(response, StructuredDecisionResponse)
+ assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
+ assert rendered["mm_processor_kwargs"] == {
+ "images_kwargs": {"size": strategy.IMAGE_SIZE},
+ "use_audio_in_video": True,
+ }
+
+ # A video without a decodable soundtrack is watched silently.
+ response = await serving.create_decision(
+ StructuredDecisionRequest(
+ **JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"]
+ )
+ )
+ assert isinstance(response, StructuredDecisionResponse)
+ assert rendered["multi_modal_data"] == {"video": [video]}
+ assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
+
+ # Audio clips cannot be read with videos that have soundtracks.
+ response = await serving.create_decision(
+ StructuredDecisionRequest(
+ **JOINT_SCHEMA_REQUEST,
+ videos=["https://example.org/talking.mp4"],
+ audio=["https://example.org/clip.wav"],
+ )
+ )
+ assert isinstance(response, ErrorResponse)
+ assert response.error.code == 400
diff --git a/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py b/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py
index da04d0108f..7aca4b64ab 100644
--- a/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py
+++ b/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
]
+
+
+def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
+ """A source with explicit allow patterns (e.g. a standalone head file in
+ the repo root) must not be validated against the backbone's index."""
+ save_file(
+ {"backbone.weight": torch.zeros(1)},
+ tmp_path / "model-00001-of-00001.safetensors",
+ )
+ save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
+ (tmp_path / "model.safetensors.index.json").write_text(
+ json.dumps(
+ {"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
+ )
+ )
+
+ loader = DefaultModelLoader(LoadConfig())
+ secondary = DefaultModelLoader.Source(
+ model_or_path=str(tmp_path),
+ revision=None,
+ prefix="pooler.head.",
+ allow_patterns_overrides=["joint_head.safetensors"],
+ )
+ assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
+ "pooler.head.head.weight"
+ ]
+ # The primary source still filters files the backbone index doesn't list.
+ primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
+ assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
diff --git a/tests/models/multimodal/pooling/test_clef_omni.py b/tests/models/multimodal/pooling/test_clef_omni.py
new file mode 100644
index 0000000000..ccdc913b31
--- /dev/null
+++ b/tests/models/multimodal/pooling/test_clef_omni.py
@@ -0,0 +1,168 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
+which loops over questions where the pooler batches them."""
+
+import math
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
+from vllm.pooling_params import PoolingParams
+from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
+
+HIDDEN, VOCAB = 64, 500
+TYPES = [0, 1, 1, 2]
+COUNTS = [2, 3, 1, 5]
+
+
+def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
+ """``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
+ for one record, with ``spans`` the option spans of each question."""
+ hidden = head.hidden_norm(hidden)
+ memory = head.memory_projection(hidden).unsqueeze(0)
+ global_vector = hidden[-1]
+ questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
+ contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
+ lexicals = [
+ torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
+ ]
+ options = torch.cat(
+ [
+ head.option_context_projection(c)
+ + head.option_lexical_projection(lex)
+ + head.option_question_projection(questions[i]).unsqueeze(0)
+ for i, (c, lex) in enumerate(zip(contexts, lexicals))
+ ]
+ ).unsqueeze(0)
+ for layer in head.evidence_layers:
+ options = layer(options, memory)
+ routed = torch.split(options[0], [len(q) for q in spans])
+
+ base = head.question_projection(questions)
+ summaries = []
+ for field, opts in zip(base, routed):
+ weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
+ summaries.append((weights[:, None] * opts).sum(0))
+ fields = (
+ base
+ + head.option_summary_norm(torch.stack(summaries))
+ + head.global_projection(global_vector)
+ + head.type_embedding(torch.tensor(TYPES))
+ ).unsqueeze(0)
+ for layer in head.layers:
+ fields = layer(fields, memory)
+ fields = head.field_norm(fields[0])
+
+ logits = []
+ for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
+ anchor = F.normalize(questions[i] + global_vector, dim=-1)
+ prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
+ prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
+ opts = head.option_norm(opts)
+ field = field.expand_as(opts)
+ features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
+ joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
+ joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
+ joint = joint + head.residual_scorer(features).squeeze(-1)
+ logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
+ return logits
+
+
+@pytest.fixture
+def pooler():
+ torch.manual_seed(0)
+ head = JointSchemaHead(
+ hidden_size=HIDDEN,
+ width=32,
+ routing_layers=2,
+ layers=2,
+ heads=4,
+ feedforward=48,
+ ).eval()
+ with torch.no_grad():
+ for p in head.parameters():
+ p.add_(torch.randn_like(p) * 0.2)
+ embedding = torch.randn(VOCAB, HIDDEN)
+ config = SimpleNamespace(
+ scheduler_config=SimpleNamespace(
+ enable_chunked_prefill=False, async_scheduling=False
+ )
+ )
+ with patch(
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
+ return_value=config,
+ ):
+ pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
+ return pooler, embedding
+
+
+def schema(length: int):
+ """Questions after a 20-token state, with options of 1 to 4 tokens."""
+ pos = 20
+ question_spans: list[tuple[int, int]] = []
+ spans: list[list[tuple[int, int]]] = []
+ for count in COUNTS:
+ question_spans.append((pos, pos + 4))
+ pos += 6
+ spans.append([])
+ for k in range(count):
+ n = 1 + (k * 3) % 4
+ spans[-1].append((pos, pos + n))
+ pos += n + 1
+ assert pos < length
+ extra = {
+ "question_types": TYPES,
+ "question_spans": [[s - length, e - length] for s, e in question_spans],
+ "option_spans": [[s - length, e - length] for q in spans for s, e in q],
+ "option_counts": COUNTS,
+ }
+ return {"joint_schema": extra}, question_spans, spans
+
+
+@torch.inference_mode()
+def test_pooler_matches_reference(pooler):
+ pooler, embedding = pooler
+ lengths = [120, 90, 100]
+ token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
+ hidden = torch.randn(sum(lengths), HIDDEN)
+ first, first_questions, first_spans = schema(lengths[0])
+ last, last_questions, last_spans = schema(lengths[2])
+ params = [
+ PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
+ # A prompt without a schema, like the profiling run's.
+ PoolingParams(task="token_classify", use_activation=True),
+ PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
+ ]
+ padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
+ for i, ids in enumerate(token_ids):
+ padded[i, : len(ids)] = ids
+ metadata = PoolingMetadata(
+ prompt_lens=torch.tensor(lengths),
+ prompt_token_ids=None,
+ prompt_token_ids_cpu=padded,
+ pooling_params=params,
+ pooling_states=[PoolingStates() for _ in lengths],
+ )
+ metadata.build_pooling_cursor(
+ torch.tensor(lengths).numpy(),
+ seq_lens_cpu=torch.tensor(lengths),
+ device=torch.device("cpu"),
+ )
+ probs, empty, logits = pooler(hidden, metadata)
+
+ first_hidden, _, last_hidden = torch.split(hidden, lengths)
+ expected = reference_logits(
+ pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
+ )
+ expected_probs = torch.cat([q.softmax(-1) for q in expected])
+ torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
+ assert empty.numel() == 0
+ expected = reference_logits(
+ pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
+ )
+ torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
diff --git a/tests/models/multimodal/processing/test_qwen3_omni.py b/tests/models/multimodal/processing/test_qwen3_omni.py
index 301471f45a..c0ebf02971 100644
--- a/tests/models/multimodal/processing/test_qwen3_omni.py
+++ b/tests/models/multimodal/processing/test_qwen3_omni.py
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
f"Got short={short_tokens}, long={long_tokens}"
)
+
+
+@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
+def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
+ """Audio and video tokens interleave by the fps the video was processed
+ at, as in HF, not by the default of one frame a second."""
+ ctx = build_model_context(
+ model_id,
+ limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
+ )
+ processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
+ rng = np.random.RandomState(42)
+ video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
+ audio = rng.rand(32000).astype(np.float32)
+ prompt = "<|vision_start|><|video_pad|><|vision_end|>"
+ mm_kwargs: dict[str, Any] = {
+ "use_audio_in_video": True,
+ "videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
+ }
+ processed = processor(
+ prompt,
+ mm_items=processor.info.parse_mm_data(
+ {"video": [video], "audio": [(audio, 16000)]}
+ ),
+ hf_processor_mm_kwargs=mm_kwargs,
+ )
+ expected = processor.info.get_hf_processor()(
+ text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
+ )
+ assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
diff --git a/tests/models/registry.py b/tests/models/registry.py
index 2ed6d4871c..0a7027461e 100644
--- a/tests/models/registry.py
+++ b/tests/models/registry.py
@@ -1351,6 +1351,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
),
+ "ClefOmniForJointSchema": _HfExamplesInfo(
+ "Cloudflare/clef-omni",
+ max_model_len=4096,
+ ),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py
index 407bffed20..0066d239ea 100644
--- a/tests/multimodal/test_video.py
+++ b/tests/multimodal/test_video.py
@@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
+ ClefOmniVideoBackend,
DynamicVideoBackend,
Glm5NextVideoBackend,
GLM46VVideoBackend,
@@ -1813,3 +1814,18 @@ def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# One initial seek, then pure walking -- no re-seek churn.
assert cap.seeks == 1
+
+
+def test_clef_omni_backend_samples_reference_frames(tmp_path):
+ """Clef-Omni samples the first frame at or after each 0.5 s boundary
+ (the reference's 2 fps timestamp sampling), not uniform strides."""
+ pytest.importorskip("cv2")
+
+ # 2 s at 30 fps: the reference reads frames 0, 15, 30 and 45, where
+ # uniform sampling would read 0, 19, 39 and 59.
+ path = _write_gray_video(tmp_path, total_frames=60, fps=30)
+ frames, metadata = ClefOmniVideoBackend.load_bytes(path.read_bytes())
+ assert metadata["frames_indices"] == [0, 15, 30, 45]
+ assert metadata["do_sample_frames"] is False
+ for frame, idx in zip(frames, [0, 15, 30, 45]):
+ assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py
index 6bf8d69026..9bbe4e879b 100644
--- a/vllm/entrypoints/generate/api_router.py
+++ b/vllm/entrypoints/generate/api_router.py
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any
from fastapi import FastAPI
import vllm.envs as envs
-from vllm.logger import init_logger
if TYPE_CHECKING:
from argparse import Namespace
@@ -18,8 +17,6 @@ if TYPE_CHECKING:
else:
RequestLogger = object
-logger = init_logger(__name__)
-
def register_generate_api_routers(app: FastAPI):
from vllm.entrypoints.openai.chat_completion.api_router import (
@@ -56,12 +53,6 @@ def register_generate_api_routers(app: FastAPI):
register_generative_scoring_api_router(app)
- from .structured_decisions.api_router import (
- register_structured_decisions_api_router,
- )
-
- register_structured_decisions_api_router(app)
-
async def init_generate_state(
engine_client: "EngineClient",
@@ -231,33 +222,3 @@ async def init_generate_state(
state.openai_serving_models,
request_logger=request_logger,
)
-
- from .structured_decisions.serving import ServingStructuredDecisions
- from .structured_decisions.strategies import ReadContext, select_read_strategy
-
- strategy = None
- if "generate" in supported_tasks:
- try:
- strategy_cls = select_read_strategy(engine_client.model_config)
- strategy = strategy_cls(
- ReadContext(
- engine_client=engine_client,
- online_renderer=state.online_renderer,
- chat_template=resolved_chat_template,
- chat_template_content_format=args.chat_template_content_format,
- default_chat_template_kwargs=default_chat_template_kwargs,
- )
- )
- except ValueError as e:
- # Info, since every model without a read strategy lands here.
- logger.info("/v1/systemone is disabled: %s", e)
- strategy = None
- state.serving_structured_decisions = (
- ServingStructuredDecisions(
- state.openai_serving_models,
- strategy,
- request_logger=request_logger,
- )
- if strategy is not None
- else None
- )
diff --git a/vllm/entrypoints/generate/structured_decisions/api_router.py b/vllm/entrypoints/generate/structured_decisions/api_router.py
index 5729b58b2e..a3afe7fe11 100644
--- a/vllm/entrypoints/generate/structured_decisions/api_router.py
+++ b/vllm/entrypoints/generate/structured_decisions/api_router.py
@@ -1,19 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from argparse import Namespace
from http import HTTPStatus
+from typing import Any
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
+from starlette.datastructures import State
+from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
+from vllm.entrypoints.serve.utils.request_logger import RequestLogger
+from vllm.logger import init_logger
+from vllm.renderers.chat_utils import load_chat_template
+from vllm.tasks import SupportedTask
from .protocol import StructuredDecisionRequest
from .serving import ServingStructuredDecisions
+from .strategies import ReadContext, select_read_strategy
+
+logger = init_logger(__name__)
router = APIRouter()
@@ -50,3 +61,31 @@ async def create_structured_decision(
def register_structured_decisions_api_router(app: FastAPI):
app.include_router(router)
+
+
+def init_structured_decisions_state(
+ engine_client: EngineClient,
+ state: State,
+ args: Namespace,
+ request_logger: RequestLogger | None,
+ supported_tasks: tuple[SupportedTask, ...],
+ default_chat_template_kwargs: dict[str, Any],
+):
+ state.serving_structured_decisions = None
+ try:
+ strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
+ strategy = strategy_cls(
+ ReadContext(
+ engine_client=engine_client,
+ online_renderer=state.online_renderer,
+ chat_template=load_chat_template(args.chat_template),
+ chat_template_content_format=args.chat_template_content_format,
+ default_chat_template_kwargs=default_chat_template_kwargs,
+ )
+ )
+ except ValueError as e:
+ logger.debug("/v1/systemone is disabled: %s", e)
+ return
+ state.serving_structured_decisions = ServingStructuredDecisions(
+ state.openai_serving_models, strategy, request_logger=request_logger
+ )
diff --git a/vllm/entrypoints/generate/structured_decisions/protocol.py b/vllm/entrypoints/generate/structured_decisions/protocol.py
index ab89084dff..a8f104b8cb 100644
--- a/vllm/entrypoints/generate/structured_decisions/protocol.py
+++ b/vllm/entrypoints/generate/structured_decisions/protocol.py
@@ -30,6 +30,15 @@ class StructuredDecisionRequest(OpenAIBaseModel):
instructions: str | None = Field(
default=None, description="Context placed ahead of the questions."
)
+ images: list[str] | None = Field(
+ default=None, description="Image URLs or data URLs, for models that see."
+ )
+ audio: list[str] | None = Field(
+ default=None, description="Audio URLs or data URLs, for models that hear."
+ )
+ videos: list[str] | None = Field(
+ default=None, description="Video URLs or data URLs, for models that see."
+ )
chat_template_kwargs: dict[str, Any] | None = None
cache_salt: str | None = Field(
default=None,
diff --git a/vllm/entrypoints/generate/structured_decisions/serving.py b/vllm/entrypoints/generate/structured_decisions/serving.py
index 2633647a0b..52fdd430b1 100644
--- a/vllm/entrypoints/generate/structured_decisions/serving.py
+++ b/vllm/entrypoints/generate/structured_decisions/serving.py
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
-import json
from typing import Any
from fastapi import Request
@@ -24,10 +23,6 @@ from .strategies import DecisionLimits, ReadStrategy
logger = init_logger(__name__)
-def state_text(state: Any) -> str:
- return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
-
-
def parse_questions(
request: StructuredDecisionRequest,
limits: DecisionLimits,
@@ -72,6 +67,18 @@ class ServingStructuredDecisions(BaseServing):
self.strategy = strategy
self.limits = strategy.limits()
+ def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
+ urls = {
+ "image": request.images,
+ "audio": request.audio,
+ "video": request.videos,
+ }
+ media = {modality: items for modality, items in urls.items() if items}
+ if unsupported := media.keys() - self.strategy.modalities:
+ names = ", ".join(sorted(unsupported))
+ raise StructuredDecisionError(f"this model does not take {names} inputs")
+ return media
+
async def create_decision(
self,
request: StructuredDecisionRequest,
@@ -87,8 +94,9 @@ class ServingStructuredDecisions(BaseServing):
request_id = f"decision-{base_id}"
try:
questions = parse_questions(request, self.limits)
+ media = self._media(request)
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
- engine_client.check_admission(len(questions))
+ engine_client.check_admission(self.strategy.num_requests(questions))
trace_headers = (
None
if raw_request is None
@@ -97,13 +105,14 @@ class ServingStructuredDecisions(BaseServing):
reads = await self.strategy.read(
questions,
request.instructions,
- state_text(request.state),
+ self.strategy.render_state(request.state),
request_id=request_id,
chat_template_kwargs=request.chat_template_kwargs,
lora_request=lora_request,
priority=request.priority,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
+ media=media,
)
except StructuredDecisionError as e:
return self.create_error_response(e)
diff --git a/vllm/entrypoints/generate/structured_decisions/strategies.py b/vllm/entrypoints/generate/structured_decisions/strategies.py
index 48debdb0b3..f4365924dd 100644
--- a/vllm/entrypoints/generate/structured_decisions/strategies.py
+++ b/vllm/entrypoints/generate/structured_decisions/strategies.py
@@ -5,24 +5,40 @@
Only models in READ_STRATEGIES are currently supported.
"""
+import asyncio
import hashlib
import json
import math
import random
+import time
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
-from typing import Any
+from functools import cached_property
+from typing import Any, ClassVar, TypeVar
+import numpy as np
+
+import vllm.envs as envs
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.label_reads import next_token_label_reads
-from vllm.inputs import EngineInput, tokens_input
+from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input
+from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
+from vllm.multimodal.media import (
+ MEDIA_CONNECTOR_REGISTRY,
+ ImageMediaIO,
+ MediaConnector,
+ MediaIO,
+ VideoMediaIO,
+)
+from vllm.pooling_params import PoolingParams
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
from vllm.renderers.inputs.preprocess import extract_prompt_components
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
+from vllm.tasks import SupportedTask
from vllm.tokenizers import TokenizerLike
from .protocol import ReadPromptRequest
@@ -34,6 +50,10 @@ from .question_types import (
label_softmax,
)
+_M = TypeVar("_M")
+
+logger = init_logger(__name__)
+
@dataclass
class ReadContext:
@@ -59,13 +79,30 @@ class QuestionRead:
output_tokens: int
+def state_text(state: Any) -> str:
+ return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
+
+
class ReadStrategy(ABC):
+ #: The engine task reads run as.
+ task: ClassVar[SupportedTask] = "generate"
+ #: The modalities a request may attach media of.
+ modalities: ClassVar[frozenset[str]] = frozenset()
+
def __init__(self, context: ReadContext):
self.context = context
@abstractmethod
def limits(self) -> DecisionLimits: ...
+ def render_state(self, state: Any) -> str:
+ """The request's state as the model reads it."""
+ return state_text(state)
+
+ def num_requests(self, questions: list[Question]) -> int:
+ """The engine requests a decision takes."""
+ return len(questions)
+
@abstractmethod
async def read(
self,
@@ -79,6 +116,7 @@ class ReadStrategy(ABC):
priority: int,
cache_salt: str | None = None,
trace_headers: Mapping[str, str] | None = None,
+ media: Mapping[str, Sequence[str]] | None = None,
) -> list[QuestionRead]:
"""One read per question, in order."""
@@ -214,6 +252,7 @@ class NextTokenStrategy(ReadStrategy):
priority: int,
cache_salt: str | None = None,
trace_headers: Mapping[str, str] | None = None,
+ media: Mapping[str, Sequence[str]] | None = None,
) -> list[QuestionRead]:
ctx = self.context
read_request = self._read_request(chat_template_kwargs, cache_salt)
@@ -334,7 +373,315 @@ class DiffusionGemmaCanvasStrategy(NextTokenStrategy):
)
+class JointSchemaStrategy(ReadStrategy):
+ """Clef-Omni. The whole request is one pooling read: the state, then every
+ question with its options, as the model was trained to read them. The
+ model's joint schema head scores each option, so a read has no labels.
+
+ Mirrors ``encode_record`` in the checkpoint's ``joint_schema_model.py``.
+ Each piece is tokenized on its own, so the spans of the questions and the
+ options are known. The spans count from the end of the prompt, so they
+ hold after the media placeholders, which come before the state, expand."""
+
+ task = "token_classify"
+ modalities = frozenset({"image", "audio", "video"})
+
+ SYSTEM_PROMPT = (
+ "Read the complete state and schema. Decide every field jointly. Each "
+ "answer must be exactly one of that field's allowed options."
+ )
+ #: The reference's default bound on a prompt, in tokens.
+ MAX_LENGTH = 64000
+ TYPE_IDS = {"noul": 0, "choice": 1, "score": 2}
+ NOUL_CRITERIA = {
+ "true": "The proposition is true or the answer is yes.",
+ "false": "The proposition is false or the answer is no.",
+ }
+ PLACEHOLDERS = {
+ "image": "<|vision_start|><|image_pad|><|vision_end|>\n",
+ "audio": "<|audio_start|><|audio_pad|><|audio_end|>\n",
+ "video": "<|vision_start|><|video_pad|><|vision_end|>\n",
+ }
+ IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_048_576}
+ #: The reference samples videos at 2 fps; the "clef_omni" video backend
+ #: reproduces the reference's sampling, pixel cap and frame count cap.
+ VIDEO_FPS = 2
+
+ def __init__(self, context: ReadContext):
+ super().__init__(context)
+ self.tokenizer = context.online_renderer.renderer.get_tokenizer()
+ self.max_length = min(
+ self.MAX_LENGTH, context.engine_client.model_config.max_model_len
+ )
+ self.prefix = self._tokens(
+ f"<|im_start|>system\n{self.SYSTEM_PROMPT}<|im_end|>\n"
+ "<|im_start|>user\nSTATE:\n"
+ )
+ self.suffix = self._tokens(
+ "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n"
+ "JOINT SCHEMA DECISIONS:"
+ )
+
+ def limits(self) -> DecisionLimits:
+ return DecisionLimits(max_questions=64, max_options=len(LABELS))
+
+ def render_state(self, state: Any) -> str:
+ return self._render(state)
+
+ def num_requests(self, questions: list[Question]) -> int:
+ return 1
+
+ @staticmethod
+ def _render(value: Any) -> str:
+ if isinstance(value, str):
+ return value
+ return json.dumps(
+ value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
+ )
+
+ def _tokens(self, text: str) -> list[int]:
+ return self.tokenizer.encode(text, add_special_tokens=False)
+
+ @cached_property
+ def _connector(self) -> MediaConnector:
+ model_config = self.context.engine_client.model_config
+ return MEDIA_CONNECTOR_REGISTRY.load(
+ envs.VLLM_MEDIA_CONNECTOR,
+ media_io_kwargs=model_config.get_multimodal_config().media_io_kwargs,
+ allowed_local_media_path=model_config.allowed_local_media_path,
+ allowed_media_domains=model_config.allowed_media_domains,
+ )
+
+ async def _fetch(
+ self, urls: Sequence[str], media_io: MediaIO[_M], timeout: int
+ ) -> list[_M]:
+ return await asyncio.gather(
+ *(
+ self._connector.load_from_url_async(
+ url, media_io, fetch_timeout=timeout
+ )
+ for url in urls
+ )
+ )
+
+ async def _soundtrack(
+ self, video_url: str
+ ) -> tuple[np.ndarray, int | float] | None:
+ """The video's soundtrack, or None when it has no decodable one."""
+ try:
+ return await self._connector.fetch_audio_async(video_url)
+ except ValueError as e:
+ logger.debug("no decodable soundtrack in %s: %s", video_url, e)
+ return None
+
+ async def _render_media(
+ self, media: Mapping[str, Sequence[str]], cache_salt: str | None
+ ) -> MultiModalInput:
+ """The system prompt and the media, with the media's placeholders
+ expanded. A video's soundtrack is heard with it when every video has
+ one."""
+ connector = self._connector
+ video_io = VideoMediaIO(
+ ImageMediaIO(),
+ num_frames=-1,
+ fps=self.VIDEO_FPS,
+ video_backend="clef_omni",
+ )
+ try:
+ images, audio, videos, tracks = await asyncio.gather(
+ asyncio.gather(
+ *map(connector.fetch_image_async, media.get("image", ()))
+ ),
+ asyncio.gather(
+ *map(connector.fetch_audio_async, media.get("audio", ()))
+ ),
+ self._fetch(
+ media.get("video", ()), video_io, envs.VLLM_VIDEO_FETCH_TIMEOUT
+ ),
+ asyncio.gather(*map(self._soundtrack, media.get("video", ()))),
+ )
+ except ValueError as e:
+ raise StructuredDecisionError(f"media: {e}") from e
+
+ audio_in_video = bool(videos) and all(t is not None for t in tracks)
+ if audio_in_video and audio:
+ raise StructuredDecisionError(
+ "audio clips cannot be read with videos that have soundtracks"
+ )
+ text = (
+ self.PLACEHOLDERS["image"] * len(images)
+ + self.PLACEHOLDERS["audio"] * len(audio)
+ + self.PLACEHOLDERS["video"] * len(videos)
+ )
+ if audio_in_video:
+ audio = [track for track in tracks if track is not None]
+ mm_data: dict[str, Any] = {}
+ if images:
+ mm_data["image"] = images
+ if audio:
+ mm_data["audio"] = audio
+ if videos:
+ mm_data["video"] = videos
+ mm_kwargs: dict[str, Any] = {"images_kwargs": {"size": self.IMAGE_SIZE}}
+ if videos:
+ mm_kwargs["use_audio_in_video"] = audio_in_video
+ prompt = TokensPrompt(
+ prompt_token_ids=self.prefix + self._tokens(text),
+ multi_modal_data=mm_data,
+ mm_processor_kwargs=mm_kwargs,
+ )
+ if cache_salt is not None:
+ prompt["cache_salt"] = cache_salt
+ renderer = self.context.online_renderer.renderer
+ try:
+ engine_input = await renderer.process_for_engine_async(prompt, time.time())
+ except ValueError as e:
+ raise StructuredDecisionError(f"media: {e}") from e
+ assert engine_input["type"] == "multimodal"
+ return engine_input
+
+ def _options(self, q: Question) -> list[tuple[int, str, Any]]:
+ """Each option's index in ``q.options``, id and description, in the
+ model's order: choices sorted by name, noul true then false, score
+ levels by index."""
+ if q.type.name == "noul":
+ return [
+ (
+ i,
+ key,
+ self.NOUL_CRITERIA[key] if o.description is None else o.description,
+ )
+ for i, (key, o) in enumerate(zip(("true", "false"), q.options))
+ ]
+ if q.type.name == "choice":
+ return sorted(
+ ((i, o.name, o.description) for i, o in enumerate(q.options)),
+ key=lambda option: option[1],
+ )
+ if q.type.name == "score":
+ return [(i, str(i), o.name) for i, o in enumerate(q.options)]
+ raise StructuredDecisionError(
+ f"question {q.id!r}: this model does not answer {q.type.name!r} questions"
+ )
+
+ def encode(
+ self, questions: list[Question], state: str, prefix: list[int] | None = None
+ ) -> tuple[list[int], dict[str, list[Any]], list[list[int]]]:
+ """The prompt, the schema for the pooler, and each question's option
+ indices in the order of the output. ``prefix`` is what precedes the
+ state: the system prompt, and the media if there are any."""
+ prefix = self.prefix if prefix is None else prefix
+ schema_ids = self._tokens("\n\nSCHEMA FIELDS:\n")
+ question_spans, option_spans, orders = [], [], []
+ for n, q in enumerate(questions, start=1):
+ schema_ids += self._tokens(
+ f"\nFIELD {n}\nID: {q.id}\nTYPE: {q.type.name}\nINSTRUCTION: "
+ )
+ start = len(schema_ids)
+ schema_ids += self._tokens(q.instructions or q.id)
+ question_spans.append([start, len(schema_ids)])
+ schema_ids += self._tokens("\nALLOWED OPTIONS:\n")
+ options = self._options(q)
+ for m, (_, option_id, description) in enumerate(options, start=1):
+ schema_ids += self._tokens(f"OPTION {m}: ")
+ start = len(schema_ids)
+ semantics = {"option_id": option_id}
+ if description is not None:
+ semantics["description"] = description
+ schema_ids += self._tokens(self._render(semantics))
+ option_spans.append([start, len(schema_ids)])
+ schema_ids += self._tokens("\n")
+ schema_ids += self._tokens("END FIELD\n")
+ orders.append([i for i, _, _ in options])
+
+ fixed = len(prefix) + len(schema_ids) + len(self.suffix)
+ if fixed > self.max_length:
+ raise StructuredDecisionError(
+ f"the questions take {fixed} tokens, more than the "
+ f"{self.max_length} this model reads"
+ )
+ state_ids = self._tokens(state)[: self.max_length - fixed]
+ prompt = prefix + state_ids + schema_ids + self.suffix
+ # The schema ends len(self.suffix) tokens before the prompt does.
+ end = len(schema_ids) + len(self.suffix)
+ schema: dict[str, list[Any]] = {
+ "question_types": [self.TYPE_IDS[q.type.name] for q in questions],
+ "question_spans": [[s - end, e - end] for s, e in question_spans],
+ "option_spans": [[s - end, e - end] for s, e in option_spans],
+ "option_counts": [len(order) for order in orders],
+ }
+ return prompt, schema, orders
+
+ async def read(
+ self,
+ questions: list[Question],
+ instructions: str | None,
+ state: str,
+ *,
+ request_id: str,
+ chat_template_kwargs: dict[str, Any] | None,
+ lora_request: LoRARequest | None,
+ priority: int,
+ cache_salt: str | None = None,
+ trace_headers: Mapping[str, str] | None = None,
+ media: Mapping[str, Sequence[str]] | None = None,
+ ) -> list[QuestionRead]:
+ if instructions:
+ raise StructuredDecisionError(
+ "this model reads a fixed prompt, so it takes no instructions"
+ )
+ engine_input: EngineInput
+ if media:
+ engine_input = await self._render_media(media, cache_salt)
+ prompt, schema, orders = self.encode(
+ questions, state, engine_input["prompt_token_ids"]
+ )
+ engine_input["prompt_token_ids"] = prompt
+ else:
+ prompt, schema, orders = self.encode(questions, state)
+ engine_input = tokens_input(prompt, cache_salt=cache_salt)
+ result = None
+ async for result in self.context.engine_client.encode(
+ engine_input,
+ PoolingParams(task="token_classify", extra_kwargs={"joint_schema": schema}),
+ f"{request_id}-0",
+ lora_request=lora_request,
+ trace_headers=trace_headers,
+ priority=priority,
+ ):
+ pass
+ if result is None:
+ raise ValueError("Failed to score the questions")
+ if result.error is not None:
+ raise ValueError(f"Failed to score the questions: {result.error.message}")
+ probs = result.outputs.data.tolist()
+ if len(probs) != sum(schema["option_counts"]):
+ raise ValueError(
+ f"Expected {sum(schema['option_counts'])} option scores, "
+ f"got {len(probs)}"
+ )
+
+ reads, offset = [], 0
+ for n, order in enumerate(orders):
+ q_probs = [0.0] * len(order)
+ for i, p in zip(order, probs[offset : offset + len(order)]):
+ q_probs[i] = p
+ offset += len(order)
+ reads.append(
+ QuestionRead(
+ probs=q_probs,
+ label_mass=1.0,
+ argmax_is_label=True,
+ # One forward pass reads the prompt for every question.
+ input_tokens=len(prompt) if n == 0 else 0,
+ output_tokens=0,
+ )
+ )
+ return reads
+
+
READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
+ "ClefOmniForJointSchema": JointSchemaStrategy,
"Qwen3ForCausalLM": NextTokenStrategy,
"Qwen3_5ForConditionalGeneration": NextTokenStrategy,
"Qwen3_5MoeForConditionalGeneration": NextTokenStrategy,
@@ -347,7 +694,9 @@ READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
-def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
+def select_read_strategy(
+ model_config: ModelConfig, supported_tasks: Sequence[SupportedTask]
+) -> type[ReadStrategy]:
"""Raise ValueError when the model cannot serve structured decisions."""
strategy = READ_STRATEGIES.get(model_config.architecture)
if strategy is None:
@@ -356,7 +705,12 @@ def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
f"{model_config.architecture}. Supported architectures: "
f"{sorted(READ_STRATEGIES)}"
)
- if model_config.logprobs_mode not in LOGPROBS_MODES:
+ if strategy.task not in supported_tasks:
+ raise ValueError(
+ f"Structured decisions on {model_config.architecture} need the "
+ f"{strategy.task} task"
+ )
+ if strategy.task == "generate" and model_config.logprobs_mode not in LOGPROBS_MODES:
raise ValueError(
"Structured decisions need --logprobs-mode raw_logprobs or "
f"processed_logprobs, not {model_config.logprobs_mode}"
diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py
index ce68890717..ff87dbea49 100644
--- a/vllm/entrypoints/launchers/api_server/app_state.py
+++ b/vllm/entrypoints/launchers/api_server/app_state.py
@@ -157,6 +157,19 @@ async def init_app_state(
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
+ from vllm.entrypoints.generate.structured_decisions.api_router import (
+ init_structured_decisions_state,
+ )
+
+ init_structured_decisions_state(
+ engine_client,
+ state,
+ args,
+ request_logger,
+ supported_tasks,
+ default_chat_template_kwargs,
+ )
+
await init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
diff --git a/vllm/entrypoints/launchers/api_server/routers.py b/vllm/entrypoints/launchers/api_server/routers.py
index 40d04862af..52c995a301 100644
--- a/vllm/entrypoints/launchers/api_server/routers.py
+++ b/vllm/entrypoints/launchers/api_server/routers.py
@@ -25,6 +25,12 @@ def register_api_routers(
register_models_api_router(app)
+ from vllm.entrypoints.generate.structured_decisions.api_router import (
+ register_structured_decisions_api_router,
+ )
+
+ register_structured_decisions_api_router(app)
+
from vllm.entrypoints.serve.sagemaker.api_router import (
attach_router as register_sagemaker_api_router,
)
diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py
index b162f60b05..6bedabfdc0 100644
--- a/vllm/model_executor/model_loader/default_loader.py
+++ b/vllm/model_executor/model_loader/default_loader.py
@@ -247,9 +247,11 @@ class DefaultModelLoader(BaseModelLoader):
subfolder=subfolder,
revision=revision,
)
- hf_weights_files = filter_duplicate_safetensors_files(
- hf_weights_files, hf_folder, index_file
- )
+ # only filter the weights from index without explicit allow patterns
+ if allow_patterns_overrides is None:
+ hf_weights_files = filter_duplicate_safetensors_files(
+ hf_weights_files, hf_folder, index_file
+ )
else:
hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files)
diff --git a/vllm/model_executor/models/clef_omni.py b/vllm/model_executor/models/clef_omni.py
new file mode 100644
index 0000000000..7cd0d67687
--- /dev/null
+++ b/vllm/model_executor/models/clef_omni.py
@@ -0,0 +1,437 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Clef-Omni (https://huggingface.co/Cloudflare/clef-omni).
+
+The Qwen3-Omni thinker with a joint schema head that scores every option of
+every question in one forward pass. The head reads the final hidden states of
+the whole prompt, so it runs as a ``token_classify`` pooler.
+
+Each request carries its schema in ``PoolingParams.extra_kwargs`` under
+``"joint_schema"``:
+
+- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
+- ``question_spans``: the ``[start, end)`` of each question's instructions.
+- ``option_spans``: the ``[start, end)`` of each option, question by question.
+- ``option_counts``: the number of options of each question.
+
+Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
+so they hold after multimodal placeholders are expanded. The output has one
+value per option, in ``option_spans`` order: its probability within its
+question, or its logit with ``use_activation=False``.
+"""
+
+import math
+from collections.abc import Callable, Mapping, Set
+from dataclasses import dataclass
+from typing import Any
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from torch import nn
+from transformers.feature_extraction_utils import BatchFeature
+
+from vllm.config import VllmConfig
+from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
+from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
+from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
+from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
+from vllm.multimodal import MULTIMODAL_REGISTRY
+from vllm.tasks import PoolingTask
+from vllm.utils.torch_utils import async_tensor_h2d
+from vllm.v1.outputs import PoolerOutput
+from vllm.v1.pool.metadata import PoolingMetadata
+
+from .interfaces_base import default_pooling_type
+from .qwen3_omni_moe_thinker import (
+ Qwen3OmniMoeThinkerDummyInputsBuilder,
+ Qwen3OmniMoeThinkerForConditionalGeneration,
+ Qwen3OmniMoeThinkerMultiModalProcessor,
+ Qwen3OmniMoeThinkerProcessingInfo,
+)
+
+JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
+NUM_QUESTION_TYPES = 3
+MAX_LOGIT_SCALE = math.log(100.0)
+
+
+def segment_mean(
+ values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
+) -> torch.Tensor:
+ sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
+ sums.index_add_(0, segment_ids, values.float())
+ return (sums / lengths[:, None]).to(values.dtype)
+
+
+@dataclass
+class SchemaIndices:
+ """Where one prompt's questions and options are, on the model's device."""
+
+ positions: torch.Tensor
+ """Every token of every span, questions first."""
+ span_ids: torch.Tensor
+ """The span of each of ``positions``."""
+ span_lengths: torch.Tensor
+ """``[Q + O]``."""
+ question_types: torch.Tensor
+ """``[Q]``."""
+ option_question: torch.Tensor
+ """The question of each option, ``[O]``."""
+ option_slots: torch.Tensor
+ """Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
+ option_mask: torch.Tensor
+ """``[Q, max_options]``, true at the slots of options."""
+
+
+class EvidenceRoutingLayer(nn.Module):
+ def __init__(self, width: int, heads: int, feedforward: int) -> None:
+ super().__init__()
+ self.query_norm = nn.LayerNorm(width)
+ self.memory_norm = nn.LayerNorm(width)
+ self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
+ self.feedforward_norm = nn.LayerNorm(width)
+ # The Identity takes the place of the checkpoint's dropout.
+ self.feedforward = nn.Sequential(
+ nn.Linear(width, feedforward),
+ nn.GELU(),
+ nn.Identity(),
+ nn.Linear(feedforward, width),
+ )
+
+ def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
+ # Separate key and value tensors, like the reference, keep attention's
+ # unpacked input projections.
+ routed, _ = self.attention(
+ self.query_norm(queries),
+ self.memory_norm(memory),
+ self.memory_norm(memory),
+ need_weights=False,
+ )
+ queries = queries + routed
+ return queries + self.feedforward(self.feedforward_norm(queries))
+
+
+class JointSchemaHead(nn.Module):
+ def __init__(
+ self,
+ hidden_size: int,
+ width: int,
+ routing_layers: int,
+ layers: int,
+ heads: int,
+ feedforward: int,
+ ) -> None:
+ super().__init__()
+ self.hidden_norm = nn.LayerNorm(hidden_size)
+ self.memory_projection = nn.Linear(hidden_size, width, bias=False)
+ self.question_projection = nn.Linear(hidden_size, width, bias=False)
+ self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
+ self.global_projection = nn.Linear(hidden_size, width, bias=False)
+ self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
+ self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
+ self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
+ self.evidence_layers = nn.ModuleList(
+ EvidenceRoutingLayer(width, heads, feedforward)
+ for _ in range(routing_layers)
+ )
+ self.option_summary_norm = nn.LayerNorm(width)
+ self.layers = nn.ModuleList(
+ nn.TransformerDecoderLayer(
+ d_model=width,
+ nhead=heads,
+ dim_feedforward=feedforward,
+ dropout=0.0,
+ activation="gelu",
+ batch_first=True,
+ norm_first=True,
+ )
+ for _ in range(layers)
+ )
+ self.field_norm = nn.LayerNorm(width)
+ self.option_norm = nn.LayerNorm(width)
+ self.residual_scorer = nn.Sequential(
+ nn.Linear(width * 4, width),
+ nn.GELU(),
+ nn.Identity(),
+ nn.Linear(width, 1),
+ )
+ self.prior_logit_scale = nn.Parameter(torch.zeros(()))
+ self.joint_logit_scale = nn.Parameter(torch.zeros(()))
+ self.residual_gate = nn.Parameter(torch.zeros(()))
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ lexical: torch.Tensor,
+ schema: SchemaIndices,
+ ) -> torch.Tensor:
+ """The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
+ final hidden states and each option's ``[O, hidden]`` mean output
+ embedding."""
+ num_questions = schema.question_types.shape[0]
+ option_question = schema.option_question
+ hidden_states = self.hidden_norm(hidden_states)
+ memory = self.memory_projection(hidden_states).unsqueeze(0)
+ global_vector = hidden_states[-1]
+ means = segment_mean(
+ hidden_states[schema.positions], schema.span_ids, schema.span_lengths
+ )
+ question_vectors, contexts = means[:num_questions], means[num_questions:]
+
+ options = (
+ self.option_context_projection(contexts)
+ + self.option_lexical_projection(lexical)
+ + self.option_question_projection(question_vectors)[option_question]
+ ).unsqueeze(0)
+ for layer in self.evidence_layers:
+ options = layer(options, memory)
+ options = options[0]
+
+ # Each field summarizes its options, weighted by their match to it.
+ base_fields = self.question_projection(question_vectors)
+ width = options.shape[-1]
+ grid = options.new_zeros(schema.option_mask.numel(), width)
+ grid[schema.option_slots] = options
+ grid = grid.view(*schema.option_mask.shape, width)
+ routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
+ routing = routing.masked_fill(~schema.option_mask, float("-inf"))
+ summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
+ fields = (
+ base_fields
+ + self.option_summary_norm(summaries)
+ + self.global_projection(global_vector).unsqueeze(0)
+ + self.type_embedding(schema.question_types)
+ ).unsqueeze(0)
+ for layer in self.layers:
+ fields = layer(fields, memory)
+ fields = self.field_norm(fields[0])[option_question]
+
+ anchors = F.normalize(question_vectors + global_vector, dim=-1)
+ prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
+ prior = prior_scale * (
+ F.normalize(lexical, dim=-1) * anchors[option_question]
+ ).sum(dim=-1)
+ options = self.option_norm(options)
+ cosine = F.cosine_similarity(fields, options, dim=-1)
+ features = torch.cat(
+ [fields, options, fields * options, torch.abs(fields - options)], dim=-1
+ )
+ residual = self.residual_scorer(features).squeeze(-1)
+ joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
+ joint = joint_scale * cosine + residual
+ return prior + torch.sigmoid(self.residual_gate) * joint
+
+
+def parse_joint_schema(
+ extra_kwargs: dict[str, Any] | None, prompt_len: int
+) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
+ """The question types, the absolute spans of the questions then of the
+ options, and the option counts. None if the schema is missing or invalid."""
+ schema = (extra_kwargs or {}).get("joint_schema")
+ if not isinstance(schema, dict):
+ return None
+ try:
+ types = [int(t) for t in schema["question_types"]]
+ counts = [int(c) for c in schema["option_counts"]]
+ question_spans = [
+ (int(start) + prompt_len, int(end) + prompt_len)
+ for start, end in schema["question_spans"]
+ ]
+ option_spans = [
+ (int(start) + prompt_len, int(end) + prompt_len)
+ for start, end in schema["option_spans"]
+ ]
+ except (KeyError, TypeError, ValueError):
+ return None
+ spans = question_spans + option_spans
+ if (
+ not types
+ or len(counts) != len(types)
+ or len(question_spans) != len(types)
+ or len(option_spans) != sum(counts)
+ or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
+ or any(c < 1 for c in counts)
+ or any(not 0 <= start < end <= prompt_len for start, end in spans)
+ ):
+ return None
+ return types, spans, counts
+
+
+class JointSchemaPooler(Pooler):
+ """Runs the joint schema head on each finished prompt.
+
+ It never raises, because it runs in the engine core: a prompt without a
+ valid schema, such as a profiling dummy, gets an empty output.
+ """
+
+ def __init__(
+ self,
+ head: JointSchemaHead,
+ embed_lexical: Callable[[torch.Tensor], torch.Tensor],
+ ) -> None:
+ super().__init__()
+ self.pooling = AllPool()
+ self.head = head
+ self.embed_lexical = embed_lexical
+
+ def get_supported_tasks(self) -> Set[PoolingTask]:
+ return {"token_classify"}
+
+ def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
+ return PoolingParamsUpdate(requires_token_ids=True)
+
+ def forward(
+ self,
+ hidden_states: torch.Tensor,
+ pooling_metadata: PoolingMetadata,
+ ) -> PoolerOutput:
+ pooled = self.pooling(hidden_states, pooling_metadata)
+ prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
+ outputs: list[torch.Tensor | None] = []
+ for hidden, token_ids, params in zip(
+ pooled, prompt_token_ids, pooling_metadata.pooling_params
+ ):
+ if hidden is None:
+ outputs.append(None)
+ continue
+ parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
+ if parsed is None:
+ outputs.append(hidden.new_empty(0, dtype=torch.float32))
+ continue
+ schema, lexical_ids, lexical_span_ids = self._indices(
+ *parsed, hidden.device
+ )
+ lexical = segment_mean(
+ self.embed_lexical(
+ async_tensor_h2d(token_ids[lexical_ids], hidden.device)
+ ),
+ lexical_span_ids,
+ schema.span_lengths[len(parsed[0]) :],
+ )
+ logits = self.head(hidden, lexical, schema).float()
+ if params.use_activation is not False:
+ mask = schema.option_mask
+ grid = logits.new_full((mask.numel(),), float("-inf"))
+ grid[schema.option_slots] = logits
+ probs = grid.view(mask.shape).softmax(dim=-1)
+ logits = probs.view(-1)[schema.option_slots]
+ outputs.append(logits)
+ return outputs
+
+ @staticmethod
+ def _indices(
+ types: list[int],
+ spans: list[tuple[int, int]],
+ counts: list[int],
+ device: torch.device,
+ ) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
+ """The schema's indices, plus the option tokens' positions (on the CPU)
+ and their option spans, for the lexical embeddings."""
+ num_questions = len(types)
+ lengths = torch.tensor([end - start for start, end in spans])
+ span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
+ positions = torch.cat([torch.arange(start, end) for start, end in spans])
+ num_question_tokens = int(lengths[:num_questions].sum())
+
+ counts_cpu = torch.tensor(counts)
+ max_options = max(counts)
+ option_question = torch.repeat_interleave(
+ torch.arange(num_questions), counts_cpu
+ )
+ option_slots = torch.cat(
+ [torch.arange(count) + q * max_options for q, count in enumerate(counts)]
+ )
+ schema = SchemaIndices(
+ positions=async_tensor_h2d(positions, device),
+ span_ids=async_tensor_h2d(span_ids, device),
+ span_lengths=async_tensor_h2d(lengths, device),
+ question_types=async_tensor_h2d(types, device, dtype=torch.long),
+ option_question=async_tensor_h2d(option_question, device),
+ option_slots=async_tensor_h2d(option_slots, device),
+ option_mask=async_tensor_h2d(
+ torch.arange(max_options) < counts_cpu[:, None], device
+ ),
+ )
+ lexical_span_ids = async_tensor_h2d(
+ span_ids[num_question_tokens:] - num_questions, device
+ )
+ return schema, positions[num_question_tokens:], lexical_span_ids
+
+
+MAX_FRAME_PIXELS = 256 * 32 * 32
+
+
+def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
+ """The reference reads each video frame at at most 256 * 32 * 32 pixels,
+ in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on
+ their device (a CUDA tensor from torchcodec must not round-trip)."""
+ height, width = frames.shape[1:3]
+ if height * width <= MAX_FRAME_PIXELS:
+ return frames
+ scale = math.sqrt(MAX_FRAME_PIXELS / (width * height))
+ new_width = max(2, round(width * scale / 2) * 2)
+ new_height = max(2, round(height * scale / 2) * 2)
+ is_tensor = isinstance(frames, torch.Tensor)
+ tensor = frames if is_tensor else torch.from_numpy(frames)
+ resized = F.interpolate(
+ tensor.permute(0, 3, 1, 2).float(),
+ size=(new_height, new_width),
+ mode="bilinear",
+ )
+ if tensor.dtype == torch.uint8:
+ resized = resized.round()
+ resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous()
+ return resized if is_tensor else resized.numpy()
+
+
+def _cap_video(item: Any) -> Any:
+ if isinstance(item, tuple):
+ frames, metadata = item
+ return cap_frame_pixels(frames), metadata
+ return cap_frame_pixels(item)
+
+
+class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor):
+ """Caps video frames at the reference's per-frame pixel count."""
+
+ def _call_hf_processor(
+ self,
+ hf_data: Mapping[str, object],
+ hf_kwargs: Mapping[str, object],
+ ) -> BatchFeature:
+ if videos := hf_data.get("videos"):
+ assert isinstance(videos, list)
+ hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]}
+ return super()._call_hf_processor(hf_data, hf_kwargs)
+
+
+@default_pooling_type(tok_pooling_type="ALL")
+@MULTIMODAL_REGISTRY.register_processor(
+ ClefOmniMultiModalProcessor,
+ info=Qwen3OmniMoeThinkerProcessingInfo,
+ dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
+)
+class ClefOmniForJointSchema(Qwen3OmniMoeThinkerForConditionalGeneration):
+ is_pooling_model = True
+
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
+ model_config = vllm_config.model_config
+ head_config = model_config.hf_config.joint_head_config
+ self.pooler = JointSchemaPooler(
+ JointSchemaHead(**head_config), self._embed_lexical
+ )
+ # The head's weights ship as their own safetensors file.
+ self.secondary_weights = [
+ DefaultModelLoader.Source(
+ model_or_path=model_config.model,
+ revision=model_config.revision,
+ prefix="pooler.head.",
+ allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
+ )
+ ]
+
+ def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
+ # The head embeds options with the output embeddings. ParallelLMHead
+ # forbids lookups, so use the vocab-parallel embedding's.
+ return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py
index 4ae80a4a36..321b5c2289 100755
--- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py
+++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py
@@ -83,6 +83,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
)
+from vllm.multimodal.utils import copy_mm_embedding_modality
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -1498,8 +1499,12 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
audio_in_video_item_idx += 1
+ # Match the fps the processor used, as HF and the mrope positions do.
+ processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
- if second_per_grid_ts:
+ if isinstance(processed_second_per_grid_ts, torch.Tensor):
+ video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item()
+ elif second_per_grid_ts:
video_second_per_grid_t = second_per_grid_ts[item_idx]
else:
video_second_per_grid_t = 2.0
@@ -1904,7 +1909,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
embeddings_main, embeddings_multiscale = torch.split(
embeddings, [visual_dim, multi_dim], dim=-1
)
- multimodal_embeddings[index] = embeddings_main
+ multimodal_embeddings[index] = copy_mm_embedding_modality(
+ embeddings, embeddings_main
+ )
multimodal_embeddings_multiscale.append(embeddings_multiscale)
if not is_interleaved:
current_positions = mm_positions[
diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py
index 427648045f..36f9bd6e6f 100644
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -287,6 +287,7 @@ _REWARD_MODELS = {
_TOKEN_CLASSIFICATION_MODELS = {
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
+ "ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
"ModernBertForTokenClassification": (
"modernbert",
"ModernBertForTokenClassification",
diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py
index fc0cdc28d4..75a6c1beb1 100644
--- a/vllm/multimodal/video.py
+++ b/vllm/multimodal/video.py
@@ -1570,3 +1570,67 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
valid_frame_indices=valid_frame_indices,
)
return frames, metadata
+
+
+@VIDEO_LOADER_REGISTRY.register("clef_omni")
+class ClefOmniVideoBackend(VideoBackend):
+ """Clef-Omni's reference sampling: the first frame at or after each
+ ``1/fps`` boundary (frame i sits at ``i / original_fps`` seconds, so the
+ k-th boundary is crossed by frame ``ceil(k * original_fps / fps)``),
+ capped at 768 frames and padded to an even count by repeating the last
+ frame."""
+
+ MAX_FRAMES = 768
+
+ @classmethod
+ def compute_frames_index_to_sample(
+ cls,
+ source: VideoSourceMetadata,
+ target: VideoTargetMetadata,
+ **kwargs,
+ ) -> list[int]:
+ if source.original_fps <= 0:
+ raise ValueError("video has an unknown frame rate")
+ step = source.original_fps / target.fps
+ indices = []
+ k = 0
+ while (index := math.ceil(k * step - 1e-6)) < source.total_frames_num:
+ indices.append(index)
+ if len(indices) > cls.MAX_FRAMES:
+ raise ValueError(
+ f"video is longer than {cls.MAX_FRAMES / target.fps:.0f} seconds"
+ )
+ k += 1
+ if not indices:
+ raise ValueError("video has no decodable frames")
+ return indices
+
+ @classmethod
+ def load_bytes(
+ cls,
+ data: bytes,
+ num_frames: int = -1,
+ fps: int = 2,
+ max_duration: int = 300,
+ frame_recovery: bool = False,
+ *,
+ backend: VideoDecoderBackend = "opencv",
+ **kwargs: Any,
+ ) -> tuple[DecodedFrames, dict[str, Any]]:
+ frames, metadata = super().load_bytes(
+ data,
+ num_frames=num_frames,
+ fps=fps,
+ max_duration=max_duration,
+ frame_recovery=frame_recovery,
+ backend=backend,
+ **kwargs,
+ )
+ if len(frames) % 2:
+ last = frames[-1:]
+ if isinstance(frames, torch.Tensor):
+ frames = torch.cat([frames, last])
+ else:
+ frames = np.concatenate([frames, last])
+ metadata["frames_indices"].append(metadata["frames_indices"][-1])
+ return frames, metadata
diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py
index d79c1275b6..147b65759d 100644
--- a/vllm/transformers_utils/model_arch_config_convertor.py
+++ b/vllm/transformers_utils/model_arch_config_convertor.py
@@ -17,6 +17,10 @@ from vllm.transformers_utils.config import (
ConfigFormat,
get_safetensors_params_metadata,
)
+from vllm.transformers_utils.repo_utils import (
+ file_or_path_exists,
+ get_hf_file_to_dict,
+)
from vllm.utils.torch_utils import common_broadcastable_dtype
logger = init_logger(__name__)
@@ -643,6 +647,31 @@ class MimoV2ModelArchConfigConvertor(ModelArchConfigConvertorBase):
_strip_mimo_v2_attention_chunk_size(hf_config, hf_text_config)
+class Qwen3OmniMoeModelArchConfigConvertor(ModelArchConfigConvertorBase):
+ def __init__(
+ self,
+ hf_config: PreTrainedConfig,
+ hf_text_config: PreTrainedConfig,
+ revision: str | None = None,
+ ):
+ # Clef-Omni keeps Qwen3-Omni's config and adds a joint schema head;
+ # route to its wrapper class when the head's config file is present.
+ # Mutate hf_config (like the deepseek_v4/mimo_v2 rewrites above)
+ # because model-class resolution reads hf_config.architectures.
+ if getattr(hf_config, "architectures", None) == [
+ "Qwen3OmniMoeForConditionalGeneration"
+ ]:
+ model_path = hf_config.name_or_path
+ if model_path and file_or_path_exists(
+ model_path, "joint_head_config.json", revision
+ ):
+ hf_config.architectures = ["ClefOmniForJointSchema"]
+ hf_config.joint_head_config = get_hf_file_to_dict(
+ "joint_head_config.json", model_path, revision
+ )
+ super().__init__(hf_config, hf_text_config, revision)
+
+
class MimoV2MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
def __init__(
self,
@@ -822,6 +851,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"bailing_hybrid_v3_mtp": BailingHybridV3MTPModelArchConfigConvertor,
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
"qwen3_5_mtp": Qwen3_5MTPModelArchConfigConvertor,
+ "qwen3_omni_moe": Qwen3OmniMoeModelArchConfigConvertor,
"qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor,
"qwen4_exp_mtp": Qwen4ExpMTPModelArchConfigConvertor,
"RefinedWeb": FalconModelArchConfigConvertor,