[Model] Support Cloudflare/clef-omni on /v1/systemone (#61064)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Jiangyun Zhu
2026-10-12 21:27:34 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Isotr0py
parent 314ab8cffd
commit 092c35dd15
20 changed files with 1592 additions and 69 deletions
@@ -130,6 +130,32 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a
| options per `choice` | 26 |
| levels per `score` | 10 |
## Clef-Omni
[Clef-Omni](https://huggingface.co/Cloudflare/clef-omni) is a decision model:
a joint schema head on the Qwen3-Omni thinker scores every option of every
question in one forward pass. vLLM serves it as a pooling model.
```bash
vllm serve Cloudflare/clef-omni
```
- A request is one read. The prompt is the one the checkpoint's
`joint_schema_model.py` builds: the state, then every question with its
options. A JSON state is written as compact JSON with sorted keys.
- A state too long for the model is truncated to fit, as the reference does.
A request whose questions alone exceed 64,000 tokens, or `max_model_len`,
returns 400.
- `instructions` returns 400, since the prompt is fixed. `chat_template_kwargs`
has no effect.
- There are no labels, so `label_mass` is 1 and `confidence` is the chosen
option's probability. `usage.input_tokens` counts the prompt once, and
`output_tokens` is 0.
- `images`, `audio` and `videos` are read as the reference does: placed
before the state, videos sampled at 2 fps, and a video's soundtrack heard
with it when every video has one. Audio clips cannot be sent together with
videos that have soundtracks.
## Request fields
| field | meaning |
@@ -139,6 +165,7 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a
| `instructions` | optional context placed ahead of the questions |
| `chat_template_kwargs` | passed to the chat template, for example `{"enable_thinking": false}` |
| `cache_salt` | optional [prefix-cache salt](../../usage/security.md#cache-salting) applied to every question |
| `images`, `audio`, `videos` | optional media URLs or data URLs, for models that take them ([Clef-Omni](#clef-omni)) |
A question with any field other than `type`, `instructions` and `criteria` is
rejected with a 400.
@@ -1,12 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from io import BytesIO
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import numpy as np
import pybase64 as base64
import pytest
import torch
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
from PIL import Image
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
@@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import (
)
from vllm.entrypoints.generate.structured_decisions.serving import (
ServingStructuredDecisions,
state_text,
parse_questions,
)
from vllm.entrypoints.generate.structured_decisions.strategies import (
DiffusionGemmaCanvasStrategy,
JointSchemaStrategy,
NextTokenStrategy,
ReadContext,
reply_label_ids,
select_read_strategy,
state_text,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.register import (
init_exception_handler,
)
from vllm.inputs import mm_input
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.multimodal.media import MediaConnector, MediaWithBytes
from vllm.outputs import (
CompletionOutput,
PoolingOutput,
PoolingRequestOutput,
RequestOutput,
)
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
def test_strategy_selection():
qwen = "Qwen3ForCausalLM"
assert select_read_strategy(model(qwen)) is NextTokenStrategy
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
qwen, generate = "Qwen3ForCausalLM", ("generate",)
assert select_read_strategy(model(qwen), generate) is NextTokenStrategy
assert (
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"))
select_read_strategy(model(qwen, "processed_logprobs"), generate)
is NextTokenStrategy
)
assert (
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate)
is DiffusionGemmaCanvasStrategy
)
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
select_read_strategy(model("LlamaForCausalLM"))
select_read_strategy(model("LlamaForCausalLM"), generate)
with pytest.raises(ValueError, match="not raw_logits"):
select_read_strategy(model(qwen, "raw_logits"))
select_read_strategy(model(qwen, "raw_logits"), generate)
# A generative architecture served as a pooling model has no read strategy.
with pytest.raises(ValueError, match="need the generate task"):
select_read_strategy(model(qwen), ("embed",))
# Clef-Omni scores options with its head, so it reads no logprobs.
clef = model("ClefOmniForJointSchema", "raw_logits")
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
def test_route_is_registered_by_default():
@@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen):
@pytest.fixture
def serving_and_engine(qwen):
def engine_and_renderer(qwen):
tokenizer, _ = qwen
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
renderer = HfRenderer(
@@ -148,6 +173,19 @@ def serving_and_engine(qwen):
chat_template=None,
chat_template_content_format="auto",
)
yield engine, online_renderer
renderer.shutdown()
def serving(engine, strategy) -> ServingStructuredDecisions:
name = engine.model_config.model
models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)])
return ServingStructuredDecisions(models, strategy)
@pytest.fixture
def serving_and_engine(engine_and_renderer):
engine, online_renderer = engine_and_renderer
async def generate(prompt, params, request_id, **kwargs):
label_ids = params.logprob_token_ids
@@ -171,11 +209,7 @@ def serving_and_engine(qwen):
engine.generate.side_effect = generate
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
models = OpenAIServingModels(
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
)
yield ServingStructuredDecisions(models, strategy), engine
renderer.shutdown()
return serving(engine, strategy), engine
@pytest.mark.asyncio
@@ -256,3 +290,286 @@ def test_canvas_read():
strategy._read_input(
{"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4]
)
JOINT_SCHEMA_PROMPT = """<|im_start|>system
Read the complete state and schema. Decide every field jointly. Each answer \
must be exactly one of that field's allowed options.<|im_end|>
<|im_start|>user
STATE:
{"customer":{"tier":"gold"},"ticket":"Charged twice"}
SCHEMA FIELDS:
FIELD 1
ID: team
TYPE: choice
INSTRUCTION: Which team?
ALLOWED OPTIONS:
OPTION 1: {"description":"payments","option_id":"billing"}
OPTION 2: {"option_id":"security"}
OPTION 3: {"description":"deliveries","option_id":"shipping"}
END FIELD
FIELD 2
ID: urgent
TYPE: noul
INSTRUCTION: urgent
ALLOWED OPTIONS:
OPTION 1: {"description":"money is moving","option_id":"true"}
OPTION 2: {"description":"The proposition is false or the answer is no.",\
"option_id":"false"}
END FIELD
FIELD 3
ID: tone
TYPE: score
INSTRUCTION: How upset?
ALLOWED OPTIONS:
OPTION 1: {"description":"calm","option_id":"0"}
OPTION 2: {"description":"annoyed","option_id":"1"}
OPTION 3: {"description":"furious","option_id":"2"}
END FIELD
<|im_end|>
<|im_start|>assistant
<think>
</think>
JOINT SCHEMA DECISIONS:"""
JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
state={"ticket": "Charged twice", "customer": {"tier": "gold"}},
questions={
"team": {
"type": "choice",
"instructions": "Which team?",
"criteria": {
"shipping": "deliveries",
"billing": "payments",
"security": None,
},
},
"urgent": {"type": "noul", "criteria": {"true": "money is moving"}},
"tone": {
"type": "score",
"instructions": "How upset?",
"criteria": ["calm", "annoyed", "furious"],
},
},
)
@pytest.fixture
def joint_schema(qwen, engine_and_renderer):
tokenizer, _ = qwen
engine, online_renderer = engine_and_renderer
# The probability the head gives each option, by option id.
scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2}
scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3}
async def encode(prompt, params, request_id, **kwargs):
ids = prompt["prompt_token_ids"]
spans = params.extra_kwargs["joint_schema"]["option_spans"]
options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans]
data = torch.tensor([scores[o["option_id"]] for o in options])
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
engine.encode.side_effect = encode
strategy = JointSchemaStrategy(
ReadContext(engine, online_renderer, None, "auto", {})
)
return serving(engine, strategy), engine, tokenizer
@pytest.mark.asyncio
async def test_joint_schema_reads_every_question_at_once(joint_schema):
serving, engine, tokenizer = joint_schema
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt")
)
assert isinstance(response, StructuredDecisionResponse)
engine.encode.assert_called_once()
engine.check_admission.assert_called_once_with(1)
prompt, params = engine.encode.call_args.args[:2]
ids = prompt["prompt_token_ids"]
assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT
assert prompt["cache_salt"] == "salt"
assert params.task == "token_classify"
schema = params.extra_kwargs["joint_schema"]
assert schema["question_types"] == [1, 0, 2]
assert schema["option_counts"] == [3, 2, 3]
assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [
"Which team?",
"urgent",
"How upset?",
]
# Options are read in the model's order and answered in the request's.
team = response.answers["team"]
assert team["choice"] == "billing"
assert list(team["probabilities"]) == ["shipping", "billing", "security"]
assert team["probabilities"] == pytest.approx(
{"shipping": 0.2, "billing": 0.7, "security": 0.1}
)
assert response.answers["urgent"]["noul"] == pytest.approx(0.9)
assert response.answers["tone"]["score"] == pytest.approx(1.1)
assert response.usage.input_tokens == len(ids)
assert response.usage.output_tokens == 0
assert all(d.label_mass == 1.0 for d in response.diagnostics.values())
@pytest.mark.asyncio
async def test_joint_schema_rejects_instructions(joint_schema):
serving, engine, _ = joint_schema
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.")
)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
engine.encode.assert_not_called()
def test_joint_schema_truncates_the_state(joint_schema):
serving, _, _ = joint_schema
strategy = serving.strategy
request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST)
questions = parse_questions(request, strategy.limits())
prompt, schema, _ = strategy.encode(questions, "word " * 100)
strategy.max_length = len(prompt) - 50
truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100)
# The state loses 50 tokens and the questions stay at the end.
assert len(truncated) == strategy.max_length
assert truncated[-200:] == prompt[-200:]
assert truncated_schema == schema
strategy.max_length = 100
with pytest.raises(StructuredDecisionError, match="questions take"):
strategy.encode(questions, "word")
@pytest.mark.asyncio
async def test_media_needs_a_model_that_takes_it(serving_and_engine):
serving, engine = serving_and_engine
request = StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"]
)
response = await serving.create_decision(request)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert "does not take image inputs" in response.error.message
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
strategy.__dict__["_connector"] = MediaConnector()
image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
rendered: dict[str, Any] = {}
async def process_for_engine_async(prompt, arrival_time):
rendered.update(prompt)
# The image's placeholder expands to three tokens.
ids = [
token
for pad in prompt["prompt_token_ids"]
for token in ([pad] * 3 if pad == image_pad else [pad])
]
return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt"))
renderer = strategy.context.online_renderer.renderer
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
buffer = BytesIO()
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
)
assert isinstance(response, StructuredDecisionResponse)
placeholder = tokenizer.encode(
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
)
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE}
}
prompt = engine.encode.call_args.args[0]
assert prompt["type"] == "multimodal"
ids = prompt["prompt_token_ids"]
expanded = len(strategy.prefix) + len(placeholder) + 2
assert ids[:expanded].count(image_pad) == 3
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
assert tokenizer.decode(ids[expanded:]) == state_and_questions
assert response.answers["team"]["choice"] == "billing"
assert response.usage.input_tokens == len(ids)
@pytest.mark.asyncio
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
video = MediaWithBytes(
(np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes"
)
track = (np.zeros(160, np.float32), 16000)
rendered: dict[str, Any] = {}
class FakeConnector:
async def fetch_image_async(self, url):
raise AssertionError("no images in this test")
async def load_from_url_async(self, url, media_io, *, fetch_timeout=None):
return video
async def fetch_audio_async(self, url):
if url == "https://example.org/silent.mp4":
raise ValueError("No audio stream found.")
return track
strategy.__dict__["_connector"] = FakeConnector()
async def process_for_engine_async(prompt, arrival_time):
rendered.clear()
rendered.update(prompt)
return mm_input(prompt["prompt_token_ids"], {}, {}, {})
renderer = strategy.context.online_renderer.renderer
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"]
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE},
"use_audio_in_video": True,
}
# A video without a decodable soundtrack is watched silently.
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"]
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video]}
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
# Audio clips cannot be read with videos that have soundtracks.
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST,
videos=["https://example.org/talking.mp4"],
audio=["https://example.org/clip.wav"],
)
)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
]
def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
"""A source with explicit allow patterns (e.g. a standalone head file in
the repo root) must not be validated against the backbone's index."""
save_file(
{"backbone.weight": torch.zeros(1)},
tmp_path / "model-00001-of-00001.safetensors",
)
save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
(tmp_path / "model.safetensors.index.json").write_text(
json.dumps(
{"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
)
)
loader = DefaultModelLoader(LoadConfig())
secondary = DefaultModelLoader.Source(
model_or_path=str(tmp_path),
revision=None,
prefix="pooler.head.",
allow_patterns_overrides=["joint_head.safetensors"],
)
assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
"pooler.head.head.weight"
]
# The primary source still filters files the backbone index doesn't list.
primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
@@ -0,0 +1,168 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
which loops over questions where the pooler batches them."""
import math
from types import SimpleNamespace
from unittest.mock import patch
import pytest
import torch
import torch.nn.functional as F
from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
from vllm.pooling_params import PoolingParams
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
HIDDEN, VOCAB = 64, 500
TYPES = [0, 1, 1, 2]
COUNTS = [2, 3, 1, 5]
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
for one record, with ``spans`` the option spans of each question."""
hidden = head.hidden_norm(hidden)
memory = head.memory_projection(hidden).unsqueeze(0)
global_vector = hidden[-1]
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
lexicals = [
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
]
options = torch.cat(
[
head.option_context_projection(c)
+ head.option_lexical_projection(lex)
+ head.option_question_projection(questions[i]).unsqueeze(0)
for i, (c, lex) in enumerate(zip(contexts, lexicals))
]
).unsqueeze(0)
for layer in head.evidence_layers:
options = layer(options, memory)
routed = torch.split(options[0], [len(q) for q in spans])
base = head.question_projection(questions)
summaries = []
for field, opts in zip(base, routed):
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
summaries.append((weights[:, None] * opts).sum(0))
fields = (
base
+ head.option_summary_norm(torch.stack(summaries))
+ head.global_projection(global_vector)
+ head.type_embedding(torch.tensor(TYPES))
).unsqueeze(0)
for layer in head.layers:
fields = layer(fields, memory)
fields = head.field_norm(fields[0])
logits = []
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
anchor = F.normalize(questions[i] + global_vector, dim=-1)
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
opts = head.option_norm(opts)
field = field.expand_as(opts)
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
joint = joint + head.residual_scorer(features).squeeze(-1)
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
return logits
@pytest.fixture
def pooler():
torch.manual_seed(0)
head = JointSchemaHead(
hidden_size=HIDDEN,
width=32,
routing_layers=2,
layers=2,
heads=4,
feedforward=48,
).eval()
with torch.no_grad():
for p in head.parameters():
p.add_(torch.randn_like(p) * 0.2)
embedding = torch.randn(VOCAB, HIDDEN)
config = SimpleNamespace(
scheduler_config=SimpleNamespace(
enable_chunked_prefill=False, async_scheduling=False
)
)
with patch(
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
return_value=config,
):
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
return pooler, embedding
def schema(length: int):
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
pos = 20
question_spans: list[tuple[int, int]] = []
spans: list[list[tuple[int, int]]] = []
for count in COUNTS:
question_spans.append((pos, pos + 4))
pos += 6
spans.append([])
for k in range(count):
n = 1 + (k * 3) % 4
spans[-1].append((pos, pos + n))
pos += n + 1
assert pos < length
extra = {
"question_types": TYPES,
"question_spans": [[s - length, e - length] for s, e in question_spans],
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
"option_counts": COUNTS,
}
return {"joint_schema": extra}, question_spans, spans
@torch.inference_mode()
def test_pooler_matches_reference(pooler):
pooler, embedding = pooler
lengths = [120, 90, 100]
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
hidden = torch.randn(sum(lengths), HIDDEN)
first, first_questions, first_spans = schema(lengths[0])
last, last_questions, last_spans = schema(lengths[2])
params = [
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
# A prompt without a schema, like the profiling run's.
PoolingParams(task="token_classify", use_activation=True),
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
]
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
for i, ids in enumerate(token_ids):
padded[i, : len(ids)] = ids
metadata = PoolingMetadata(
prompt_lens=torch.tensor(lengths),
prompt_token_ids=None,
prompt_token_ids_cpu=padded,
pooling_params=params,
pooling_states=[PoolingStates() for _ in lengths],
)
metadata.build_pooling_cursor(
torch.tensor(lengths).numpy(),
seq_lens_cpu=torch.tensor(lengths),
device=torch.device("cpu"),
)
probs, empty, logits = pooler(hidden, metadata)
first_hidden, _, last_hidden = torch.split(hidden, lengths)
expected = reference_logits(
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
)
expected_probs = torch.cat([q.softmax(-1) for q in expected])
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
assert empty.numel() == 0
expected = reference_logits(
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
)
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
f"Got short={short_tokens}, long={long_tokens}"
)
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
"""Audio and video tokens interleave by the fps the video was processed
at, as in HF, not by the default of one frame a second."""
ctx = build_model_context(
model_id,
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
)
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
rng = np.random.RandomState(42)
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
audio = rng.rand(32000).astype(np.float32)
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
mm_kwargs: dict[str, Any] = {
"use_audio_in_video": True,
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
}
processed = processor(
prompt,
mm_items=processor.info.parse_mm_data(
{"video": [video], "audio": [(audio, 16000)]}
),
hf_processor_mm_kwargs=mm_kwargs,
)
expected = processor.info.get_hf_processor()(
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
)
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
+4
View File
@@ -1351,6 +1351,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
max_model_len=4096,
),
"ClefOmniForJointSchema": _HfExamplesInfo(
"Cloudflare/clef-omni",
max_model_len=4096,
),
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
"Qwen/Qwen3-ASR-0.6B",
max_model_len=4096,
+16
View File
@@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
from vllm.multimodal.video import (
PYNVVIDEOCODEC_VIDEO_BACKEND,
VIDEO_LOADER_REGISTRY,
ClefOmniVideoBackend,
DynamicVideoBackend,
Glm5NextVideoBackend,
GLM46VVideoBackend,
@@ -1813,3 +1814,18 @@ def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
# One initial seek, then pure walking -- no re-seek churn.
assert cap.seeks == 1
def test_clef_omni_backend_samples_reference_frames(tmp_path):
"""Clef-Omni samples the first frame at or after each 0.5 s boundary
(the reference's 2 fps timestamp sampling), not uniform strides."""
pytest.importorskip("cv2")
# 2 s at 30 fps: the reference reads frames 0, 15, 30 and 45, where
# uniform sampling would read 0, 19, 39 and 59.
path = _write_gray_video(tmp_path, total_frames=60, fps=30)
frames, metadata = ClefOmniVideoBackend.load_bytes(path.read_bytes())
assert metadata["frames_indices"] == [0, 15, 30, 45]
assert metadata["do_sample_frames"] is False
for frame, idx in zip(frames, [0, 15, 30, 45]):
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
-39
View File
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any
from fastapi import FastAPI
import vllm.envs as envs
from vllm.logger import init_logger
if TYPE_CHECKING:
from argparse import Namespace
@@ -18,8 +17,6 @@ if TYPE_CHECKING:
else:
RequestLogger = object
logger = init_logger(__name__)
def register_generate_api_routers(app: FastAPI):
from vllm.entrypoints.openai.chat_completion.api_router import (
@@ -56,12 +53,6 @@ def register_generate_api_routers(app: FastAPI):
register_generative_scoring_api_router(app)
from .structured_decisions.api_router import (
register_structured_decisions_api_router,
)
register_structured_decisions_api_router(app)
async def init_generate_state(
engine_client: "EngineClient",
@@ -231,33 +222,3 @@ async def init_generate_state(
state.openai_serving_models,
request_logger=request_logger,
)
from .structured_decisions.serving import ServingStructuredDecisions
from .structured_decisions.strategies import ReadContext, select_read_strategy
strategy = None
if "generate" in supported_tasks:
try:
strategy_cls = select_read_strategy(engine_client.model_config)
strategy = strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
)
except ValueError as e:
# Info, since every model without a read strategy lands here.
logger.info("/v1/systemone is disabled: %s", e)
strategy = None
state.serving_structured_decisions = (
ServingStructuredDecisions(
state.openai_serving_models,
strategy,
request_logger=request_logger,
)
if strategy is not None
else None
)
@@ -1,19 +1,30 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from http import HTTPStatus
from typing import Any
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from vllm.renderers.chat_utils import load_chat_template
from vllm.tasks import SupportedTask
from .protocol import StructuredDecisionRequest
from .serving import ServingStructuredDecisions
from .strategies import ReadContext, select_read_strategy
logger = init_logger(__name__)
router = APIRouter()
@@ -50,3 +61,31 @@ async def create_structured_decision(
def register_structured_decisions_api_router(app: FastAPI):
app.include_router(router)
def init_structured_decisions_state(
engine_client: EngineClient,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
supported_tasks: tuple[SupportedTask, ...],
default_chat_template_kwargs: dict[str, Any],
):
state.serving_structured_decisions = None
try:
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
strategy = strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
chat_template=load_chat_template(args.chat_template),
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
)
except ValueError as e:
logger.debug("/v1/systemone is disabled: %s", e)
return
state.serving_structured_decisions = ServingStructuredDecisions(
state.openai_serving_models, strategy, request_logger=request_logger
)
@@ -30,6 +30,15 @@ class StructuredDecisionRequest(OpenAIBaseModel):
instructions: str | None = Field(
default=None, description="Context placed ahead of the questions."
)
images: list[str] | None = Field(
default=None, description="Image URLs or data URLs, for models that see."
)
audio: list[str] | None = Field(
default=None, description="Audio URLs or data URLs, for models that hear."
)
videos: list[str] | None = Field(
default=None, description="Video URLs or data URLs, for models that see."
)
chat_template_kwargs: dict[str, Any] | None = None
cache_salt: str | None = Field(
default=None,
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
from typing import Any
from fastapi import Request
@@ -24,10 +23,6 @@ from .strategies import DecisionLimits, ReadStrategy
logger = init_logger(__name__)
def state_text(state: Any) -> str:
return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
def parse_questions(
request: StructuredDecisionRequest,
limits: DecisionLimits,
@@ -72,6 +67,18 @@ class ServingStructuredDecisions(BaseServing):
self.strategy = strategy
self.limits = strategy.limits()
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
urls = {
"image": request.images,
"audio": request.audio,
"video": request.videos,
}
media = {modality: items for modality, items in urls.items() if items}
if unsupported := media.keys() - self.strategy.modalities:
names = ", ".join(sorted(unsupported))
raise StructuredDecisionError(f"this model does not take {names} inputs")
return media
async def create_decision(
self,
request: StructuredDecisionRequest,
@@ -87,8 +94,9 @@ class ServingStructuredDecisions(BaseServing):
request_id = f"decision-{base_id}"
try:
questions = parse_questions(request, self.limits)
media = self._media(request)
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine_client.check_admission(len(questions))
engine_client.check_admission(self.strategy.num_requests(questions))
trace_headers = (
None
if raw_request is None
@@ -97,13 +105,14 @@ class ServingStructuredDecisions(BaseServing):
reads = await self.strategy.read(
questions,
request.instructions,
state_text(request.state),
self.strategy.render_state(request.state),
request_id=request_id,
chat_template_kwargs=request.chat_template_kwargs,
lora_request=lora_request,
priority=request.priority,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
media=media,
)
except StructuredDecisionError as e:
return self.create_error_response(e)
@@ -5,24 +5,40 @@
Only models in READ_STRATEGIES are currently supported.
"""
import asyncio
import hashlib
import json
import math
import random
import time
from abc import ABC, abstractmethod
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from typing import Any
from functools import cached_property
from typing import Any, ClassVar, TypeVar
import numpy as np
import vllm.envs as envs
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.label_reads import next_token_label_reads
from vllm.inputs import EngineInput, tokens_input
from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input
from vllm.logger import init_logger
from vllm.lora.request import LoRARequest
from vllm.multimodal.media import (
MEDIA_CONNECTOR_REGISTRY,
ImageMediaIO,
MediaConnector,
MediaIO,
VideoMediaIO,
)
from vllm.pooling_params import PoolingParams
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
from vllm.renderers.inputs.preprocess import extract_prompt_components
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.tasks import SupportedTask
from vllm.tokenizers import TokenizerLike
from .protocol import ReadPromptRequest
@@ -34,6 +50,10 @@ from .question_types import (
label_softmax,
)
_M = TypeVar("_M")
logger = init_logger(__name__)
@dataclass
class ReadContext:
@@ -59,13 +79,30 @@ class QuestionRead:
output_tokens: int
def state_text(state: Any) -> str:
return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
class ReadStrategy(ABC):
#: The engine task reads run as.
task: ClassVar[SupportedTask] = "generate"
#: The modalities a request may attach media of.
modalities: ClassVar[frozenset[str]] = frozenset()
def __init__(self, context: ReadContext):
self.context = context
@abstractmethod
def limits(self) -> DecisionLimits: ...
def render_state(self, state: Any) -> str:
"""The request's state as the model reads it."""
return state_text(state)
def num_requests(self, questions: list[Question]) -> int:
"""The engine requests a decision takes."""
return len(questions)
@abstractmethod
async def read(
self,
@@ -79,6 +116,7 @@ class ReadStrategy(ABC):
priority: int,
cache_salt: str | None = None,
trace_headers: Mapping[str, str] | None = None,
media: Mapping[str, Sequence[str]] | None = None,
) -> list[QuestionRead]:
"""One read per question, in order."""
@@ -214,6 +252,7 @@ class NextTokenStrategy(ReadStrategy):
priority: int,
cache_salt: str | None = None,
trace_headers: Mapping[str, str] | None = None,
media: Mapping[str, Sequence[str]] | None = None,
) -> list[QuestionRead]:
ctx = self.context
read_request = self._read_request(chat_template_kwargs, cache_salt)
@@ -334,7 +373,315 @@ class DiffusionGemmaCanvasStrategy(NextTokenStrategy):
)
class JointSchemaStrategy(ReadStrategy):
"""Clef-Omni. The whole request is one pooling read: the state, then every
question with its options, as the model was trained to read them. The
model's joint schema head scores each option, so a read has no labels.
Mirrors ``encode_record`` in the checkpoint's ``joint_schema_model.py``.
Each piece is tokenized on its own, so the spans of the questions and the
options are known. The spans count from the end of the prompt, so they
hold after the media placeholders, which come before the state, expand."""
task = "token_classify"
modalities = frozenset({"image", "audio", "video"})
SYSTEM_PROMPT = (
"Read the complete state and schema. Decide every field jointly. Each "
"answer must be exactly one of that field's allowed options."
)
#: The reference's default bound on a prompt, in tokens.
MAX_LENGTH = 64000
TYPE_IDS = {"noul": 0, "choice": 1, "score": 2}
NOUL_CRITERIA = {
"true": "The proposition is true or the answer is yes.",
"false": "The proposition is false or the answer is no.",
}
PLACEHOLDERS = {
"image": "<|vision_start|><|image_pad|><|vision_end|>\n",
"audio": "<|audio_start|><|audio_pad|><|audio_end|>\n",
"video": "<|vision_start|><|video_pad|><|vision_end|>\n",
}
IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_048_576}
#: The reference samples videos at 2 fps; the "clef_omni" video backend
#: reproduces the reference's sampling, pixel cap and frame count cap.
VIDEO_FPS = 2
def __init__(self, context: ReadContext):
super().__init__(context)
self.tokenizer = context.online_renderer.renderer.get_tokenizer()
self.max_length = min(
self.MAX_LENGTH, context.engine_client.model_config.max_model_len
)
self.prefix = self._tokens(
f"<|im_start|>system\n{self.SYSTEM_PROMPT}<|im_end|>\n"
"<|im_start|>user\nSTATE:\n"
)
self.suffix = self._tokens(
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
"JOINT SCHEMA DECISIONS:"
)
def limits(self) -> DecisionLimits:
return DecisionLimits(max_questions=64, max_options=len(LABELS))
def render_state(self, state: Any) -> str:
return self._render(state)
def num_requests(self, questions: list[Question]) -> int:
return 1
@staticmethod
def _render(value: Any) -> str:
if isinstance(value, str):
return value
return json.dumps(
value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
)
def _tokens(self, text: str) -> list[int]:
return self.tokenizer.encode(text, add_special_tokens=False)
@cached_property
def _connector(self) -> MediaConnector:
model_config = self.context.engine_client.model_config
return MEDIA_CONNECTOR_REGISTRY.load(
envs.VLLM_MEDIA_CONNECTOR,
media_io_kwargs=model_config.get_multimodal_config().media_io_kwargs,
allowed_local_media_path=model_config.allowed_local_media_path,
allowed_media_domains=model_config.allowed_media_domains,
)
async def _fetch(
self, urls: Sequence[str], media_io: MediaIO[_M], timeout: int
) -> list[_M]:
return await asyncio.gather(
*(
self._connector.load_from_url_async(
url, media_io, fetch_timeout=timeout
)
for url in urls
)
)
async def _soundtrack(
self, video_url: str
) -> tuple[np.ndarray, int | float] | None:
"""The video's soundtrack, or None when it has no decodable one."""
try:
return await self._connector.fetch_audio_async(video_url)
except ValueError as e:
logger.debug("no decodable soundtrack in %s: %s", video_url, e)
return None
async def _render_media(
self, media: Mapping[str, Sequence[str]], cache_salt: str | None
) -> MultiModalInput:
"""The system prompt and the media, with the media's placeholders
expanded. A video's soundtrack is heard with it when every video has
one."""
connector = self._connector
video_io = VideoMediaIO(
ImageMediaIO(),
num_frames=-1,
fps=self.VIDEO_FPS,
video_backend="clef_omni",
)
try:
images, audio, videos, tracks = await asyncio.gather(
asyncio.gather(
*map(connector.fetch_image_async, media.get("image", ()))
),
asyncio.gather(
*map(connector.fetch_audio_async, media.get("audio", ()))
),
self._fetch(
media.get("video", ()), video_io, envs.VLLM_VIDEO_FETCH_TIMEOUT
),
asyncio.gather(*map(self._soundtrack, media.get("video", ()))),
)
except ValueError as e:
raise StructuredDecisionError(f"media: {e}") from e
audio_in_video = bool(videos) and all(t is not None for t in tracks)
if audio_in_video and audio:
raise StructuredDecisionError(
"audio clips cannot be read with videos that have soundtracks"
)
text = (
self.PLACEHOLDERS["image"] * len(images)
+ self.PLACEHOLDERS["audio"] * len(audio)
+ self.PLACEHOLDERS["video"] * len(videos)
)
if audio_in_video:
audio = [track for track in tracks if track is not None]
mm_data: dict[str, Any] = {}
if images:
mm_data["image"] = images
if audio:
mm_data["audio"] = audio
if videos:
mm_data["video"] = videos
mm_kwargs: dict[str, Any] = {"images_kwargs": {"size": self.IMAGE_SIZE}}
if videos:
mm_kwargs["use_audio_in_video"] = audio_in_video
prompt = TokensPrompt(
prompt_token_ids=self.prefix + self._tokens(text),
multi_modal_data=mm_data,
mm_processor_kwargs=mm_kwargs,
)
if cache_salt is not None:
prompt["cache_salt"] = cache_salt
renderer = self.context.online_renderer.renderer
try:
engine_input = await renderer.process_for_engine_async(prompt, time.time())
except ValueError as e:
raise StructuredDecisionError(f"media: {e}") from e
assert engine_input["type"] == "multimodal"
return engine_input
def _options(self, q: Question) -> list[tuple[int, str, Any]]:
"""Each option's index in ``q.options``, id and description, in the
model's order: choices sorted by name, noul true then false, score
levels by index."""
if q.type.name == "noul":
return [
(
i,
key,
self.NOUL_CRITERIA[key] if o.description is None else o.description,
)
for i, (key, o) in enumerate(zip(("true", "false"), q.options))
]
if q.type.name == "choice":
return sorted(
((i, o.name, o.description) for i, o in enumerate(q.options)),
key=lambda option: option[1],
)
if q.type.name == "score":
return [(i, str(i), o.name) for i, o in enumerate(q.options)]
raise StructuredDecisionError(
f"question {q.id!r}: this model does not answer {q.type.name!r} questions"
)
def encode(
self, questions: list[Question], state: str, prefix: list[int] | None = None
) -> tuple[list[int], dict[str, list[Any]], list[list[int]]]:
"""The prompt, the schema for the pooler, and each question's option
indices in the order of the output. ``prefix`` is what precedes the
state: the system prompt, and the media if there are any."""
prefix = self.prefix if prefix is None else prefix
schema_ids = self._tokens("\n\nSCHEMA FIELDS:\n")
question_spans, option_spans, orders = [], [], []
for n, q in enumerate(questions, start=1):
schema_ids += self._tokens(
f"\nFIELD {n}\nID: {q.id}\nTYPE: {q.type.name}\nINSTRUCTION: "
)
start = len(schema_ids)
schema_ids += self._tokens(q.instructions or q.id)
question_spans.append([start, len(schema_ids)])
schema_ids += self._tokens("\nALLOWED OPTIONS:\n")
options = self._options(q)
for m, (_, option_id, description) in enumerate(options, start=1):
schema_ids += self._tokens(f"OPTION {m}: ")
start = len(schema_ids)
semantics = {"option_id": option_id}
if description is not None:
semantics["description"] = description
schema_ids += self._tokens(self._render(semantics))
option_spans.append([start, len(schema_ids)])
schema_ids += self._tokens("\n")
schema_ids += self._tokens("END FIELD\n")
orders.append([i for i, _, _ in options])
fixed = len(prefix) + len(schema_ids) + len(self.suffix)
if fixed > self.max_length:
raise StructuredDecisionError(
f"the questions take {fixed} tokens, more than the "
f"{self.max_length} this model reads"
)
state_ids = self._tokens(state)[: self.max_length - fixed]
prompt = prefix + state_ids + schema_ids + self.suffix
# The schema ends len(self.suffix) tokens before the prompt does.
end = len(schema_ids) + len(self.suffix)
schema: dict[str, list[Any]] = {
"question_types": [self.TYPE_IDS[q.type.name] for q in questions],
"question_spans": [[s - end, e - end] for s, e in question_spans],
"option_spans": [[s - end, e - end] for s, e in option_spans],
"option_counts": [len(order) for order in orders],
}
return prompt, schema, orders
async def read(
self,
questions: list[Question],
instructions: str | None,
state: str,
*,
request_id: str,
chat_template_kwargs: dict[str, Any] | None,
lora_request: LoRARequest | None,
priority: int,
cache_salt: str | None = None,
trace_headers: Mapping[str, str] | None = None,
media: Mapping[str, Sequence[str]] | None = None,
) -> list[QuestionRead]:
if instructions:
raise StructuredDecisionError(
"this model reads a fixed prompt, so it takes no instructions"
)
engine_input: EngineInput
if media:
engine_input = await self._render_media(media, cache_salt)
prompt, schema, orders = self.encode(
questions, state, engine_input["prompt_token_ids"]
)
engine_input["prompt_token_ids"] = prompt
else:
prompt, schema, orders = self.encode(questions, state)
engine_input = tokens_input(prompt, cache_salt=cache_salt)
result = None
async for result in self.context.engine_client.encode(
engine_input,
PoolingParams(task="token_classify", extra_kwargs={"joint_schema": schema}),
f"{request_id}-0",
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
):
pass
if result is None:
raise ValueError("Failed to score the questions")
if result.error is not None:
raise ValueError(f"Failed to score the questions: {result.error.message}")
probs = result.outputs.data.tolist()
if len(probs) != sum(schema["option_counts"]):
raise ValueError(
f"Expected {sum(schema['option_counts'])} option scores, "
f"got {len(probs)}"
)
reads, offset = [], 0
for n, order in enumerate(orders):
q_probs = [0.0] * len(order)
for i, p in zip(order, probs[offset : offset + len(order)]):
q_probs[i] = p
offset += len(order)
reads.append(
QuestionRead(
probs=q_probs,
label_mass=1.0,
argmax_is_label=True,
# One forward pass reads the prompt for every question.
input_tokens=len(prompt) if n == 0 else 0,
output_tokens=0,
)
)
return reads
READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
"ClefOmniForJointSchema": JointSchemaStrategy,
"Qwen3ForCausalLM": NextTokenStrategy,
"Qwen3_5ForConditionalGeneration": NextTokenStrategy,
"Qwen3_5MoeForConditionalGeneration": NextTokenStrategy,
@@ -347,7 +694,9 @@ READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
def select_read_strategy(
model_config: ModelConfig, supported_tasks: Sequence[SupportedTask]
) -> type[ReadStrategy]:
"""Raise ValueError when the model cannot serve structured decisions."""
strategy = READ_STRATEGIES.get(model_config.architecture)
if strategy is None:
@@ -356,7 +705,12 @@ def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
f"{model_config.architecture}. Supported architectures: "
f"{sorted(READ_STRATEGIES)}"
)
if model_config.logprobs_mode not in LOGPROBS_MODES:
if strategy.task not in supported_tasks:
raise ValueError(
f"Structured decisions on {model_config.architecture} need the "
f"{strategy.task} task"
)
if strategy.task == "generate" and model_config.logprobs_mode not in LOGPROBS_MODES:
raise ValueError(
"Structured decisions need --logprobs-mode raw_logprobs or "
f"processed_logprobs, not {model_config.logprobs_mode}"
@@ -157,6 +157,19 @@ async def init_app_state(
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
from vllm.entrypoints.generate.structured_decisions.api_router import (
init_structured_decisions_state,
)
init_structured_decisions_state(
engine_client,
state,
args,
request_logger,
supported_tasks,
default_chat_template_kwargs,
)
await init_endpoint_plugins_state(engine_client, state, args)
state.enable_server_load_tracking = args.enable_server_load_tracking
@@ -25,6 +25,12 @@ def register_api_routers(
register_models_api_router(app)
from vllm.entrypoints.generate.structured_decisions.api_router import (
register_structured_decisions_api_router,
)
register_structured_decisions_api_router(app)
from vllm.entrypoints.serve.sagemaker.api_router import (
attach_router as register_sagemaker_api_router,
)
@@ -247,9 +247,11 @@ class DefaultModelLoader(BaseModelLoader):
subfolder=subfolder,
revision=revision,
)
hf_weights_files = filter_duplicate_safetensors_files(
hf_weights_files, hf_folder, index_file
)
# only filter the weights from index without explicit allow patterns
if allow_patterns_overrides is None:
hf_weights_files = filter_duplicate_safetensors_files(
hf_weights_files, hf_folder, index_file
)
else:
hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files)
+437
View File
@@ -0,0 +1,437 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Clef-Omni (https://huggingface.co/Cloudflare/clef-omni).
The Qwen3-Omni thinker with a joint schema head that scores every option of
every question in one forward pass. The head reads the final hidden states of
the whole prompt, so it runs as a ``token_classify`` pooler.
Each request carries its schema in ``PoolingParams.extra_kwargs`` under
``"joint_schema"``:
- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
- ``question_spans``: the ``[start, end)`` of each question's instructions.
- ``option_spans``: the ``[start, end)`` of each option, question by question.
- ``option_counts``: the number of options of each question.
Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
so they hold after multimodal placeholders are expanded. The output has one
value per option, in ``option_spans`` order: its probability within its
question, or its logit with ``use_activation=False``.
"""
import math
from collections.abc import Callable, Mapping, Set
from dataclasses import dataclass
from typing import Any
import numpy as np
import torch
import torch.nn.functional as F
from torch import nn
from transformers.feature_extraction_utils import BatchFeature
from vllm.config import VllmConfig
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.tasks import PoolingTask
from vllm.utils.torch_utils import async_tensor_h2d
from vllm.v1.outputs import PoolerOutput
from vllm.v1.pool.metadata import PoolingMetadata
from .interfaces_base import default_pooling_type
from .qwen3_omni_moe_thinker import (
Qwen3OmniMoeThinkerDummyInputsBuilder,
Qwen3OmniMoeThinkerForConditionalGeneration,
Qwen3OmniMoeThinkerMultiModalProcessor,
Qwen3OmniMoeThinkerProcessingInfo,
)
JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
NUM_QUESTION_TYPES = 3
MAX_LOGIT_SCALE = math.log(100.0)
def segment_mean(
values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
) -> torch.Tensor:
sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
sums.index_add_(0, segment_ids, values.float())
return (sums / lengths[:, None]).to(values.dtype)
@dataclass
class SchemaIndices:
"""Where one prompt's questions and options are, on the model's device."""
positions: torch.Tensor
"""Every token of every span, questions first."""
span_ids: torch.Tensor
"""The span of each of ``positions``."""
span_lengths: torch.Tensor
"""``[Q + O]``."""
question_types: torch.Tensor
"""``[Q]``."""
option_question: torch.Tensor
"""The question of each option, ``[O]``."""
option_slots: torch.Tensor
"""Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
option_mask: torch.Tensor
"""``[Q, max_options]``, true at the slots of options."""
class EvidenceRoutingLayer(nn.Module):
def __init__(self, width: int, heads: int, feedforward: int) -> None:
super().__init__()
self.query_norm = nn.LayerNorm(width)
self.memory_norm = nn.LayerNorm(width)
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
self.feedforward_norm = nn.LayerNorm(width)
# The Identity takes the place of the checkpoint's dropout.
self.feedforward = nn.Sequential(
nn.Linear(width, feedforward),
nn.GELU(),
nn.Identity(),
nn.Linear(feedforward, width),
)
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
# Separate key and value tensors, like the reference, keep attention's
# unpacked input projections.
routed, _ = self.attention(
self.query_norm(queries),
self.memory_norm(memory),
self.memory_norm(memory),
need_weights=False,
)
queries = queries + routed
return queries + self.feedforward(self.feedforward_norm(queries))
class JointSchemaHead(nn.Module):
def __init__(
self,
hidden_size: int,
width: int,
routing_layers: int,
layers: int,
heads: int,
feedforward: int,
) -> None:
super().__init__()
self.hidden_norm = nn.LayerNorm(hidden_size)
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
self.question_projection = nn.Linear(hidden_size, width, bias=False)
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
self.global_projection = nn.Linear(hidden_size, width, bias=False)
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
self.evidence_layers = nn.ModuleList(
EvidenceRoutingLayer(width, heads, feedforward)
for _ in range(routing_layers)
)
self.option_summary_norm = nn.LayerNorm(width)
self.layers = nn.ModuleList(
nn.TransformerDecoderLayer(
d_model=width,
nhead=heads,
dim_feedforward=feedforward,
dropout=0.0,
activation="gelu",
batch_first=True,
norm_first=True,
)
for _ in range(layers)
)
self.field_norm = nn.LayerNorm(width)
self.option_norm = nn.LayerNorm(width)
self.residual_scorer = nn.Sequential(
nn.Linear(width * 4, width),
nn.GELU(),
nn.Identity(),
nn.Linear(width, 1),
)
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
self.residual_gate = nn.Parameter(torch.zeros(()))
def forward(
self,
hidden_states: torch.Tensor,
lexical: torch.Tensor,
schema: SchemaIndices,
) -> torch.Tensor:
"""The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
final hidden states and each option's ``[O, hidden]`` mean output
embedding."""
num_questions = schema.question_types.shape[0]
option_question = schema.option_question
hidden_states = self.hidden_norm(hidden_states)
memory = self.memory_projection(hidden_states).unsqueeze(0)
global_vector = hidden_states[-1]
means = segment_mean(
hidden_states[schema.positions], schema.span_ids, schema.span_lengths
)
question_vectors, contexts = means[:num_questions], means[num_questions:]
options = (
self.option_context_projection(contexts)
+ self.option_lexical_projection(lexical)
+ self.option_question_projection(question_vectors)[option_question]
).unsqueeze(0)
for layer in self.evidence_layers:
options = layer(options, memory)
options = options[0]
# Each field summarizes its options, weighted by their match to it.
base_fields = self.question_projection(question_vectors)
width = options.shape[-1]
grid = options.new_zeros(schema.option_mask.numel(), width)
grid[schema.option_slots] = options
grid = grid.view(*schema.option_mask.shape, width)
routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
routing = routing.masked_fill(~schema.option_mask, float("-inf"))
summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
fields = (
base_fields
+ self.option_summary_norm(summaries)
+ self.global_projection(global_vector).unsqueeze(0)
+ self.type_embedding(schema.question_types)
).unsqueeze(0)
for layer in self.layers:
fields = layer(fields, memory)
fields = self.field_norm(fields[0])[option_question]
anchors = F.normalize(question_vectors + global_vector, dim=-1)
prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
prior = prior_scale * (
F.normalize(lexical, dim=-1) * anchors[option_question]
).sum(dim=-1)
options = self.option_norm(options)
cosine = F.cosine_similarity(fields, options, dim=-1)
features = torch.cat(
[fields, options, fields * options, torch.abs(fields - options)], dim=-1
)
residual = self.residual_scorer(features).squeeze(-1)
joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
joint = joint_scale * cosine + residual
return prior + torch.sigmoid(self.residual_gate) * joint
def parse_joint_schema(
extra_kwargs: dict[str, Any] | None, prompt_len: int
) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
"""The question types, the absolute spans of the questions then of the
options, and the option counts. None if the schema is missing or invalid."""
schema = (extra_kwargs or {}).get("joint_schema")
if not isinstance(schema, dict):
return None
try:
types = [int(t) for t in schema["question_types"]]
counts = [int(c) for c in schema["option_counts"]]
question_spans = [
(int(start) + prompt_len, int(end) + prompt_len)
for start, end in schema["question_spans"]
]
option_spans = [
(int(start) + prompt_len, int(end) + prompt_len)
for start, end in schema["option_spans"]
]
except (KeyError, TypeError, ValueError):
return None
spans = question_spans + option_spans
if (
not types
or len(counts) != len(types)
or len(question_spans) != len(types)
or len(option_spans) != sum(counts)
or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
or any(c < 1 for c in counts)
or any(not 0 <= start < end <= prompt_len for start, end in spans)
):
return None
return types, spans, counts
class JointSchemaPooler(Pooler):
"""Runs the joint schema head on each finished prompt.
It never raises, because it runs in the engine core: a prompt without a
valid schema, such as a profiling dummy, gets an empty output.
"""
def __init__(
self,
head: JointSchemaHead,
embed_lexical: Callable[[torch.Tensor], torch.Tensor],
) -> None:
super().__init__()
self.pooling = AllPool()
self.head = head
self.embed_lexical = embed_lexical
def get_supported_tasks(self) -> Set[PoolingTask]:
return {"token_classify"}
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
return PoolingParamsUpdate(requires_token_ids=True)
def forward(
self,
hidden_states: torch.Tensor,
pooling_metadata: PoolingMetadata,
) -> PoolerOutput:
pooled = self.pooling(hidden_states, pooling_metadata)
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
outputs: list[torch.Tensor | None] = []
for hidden, token_ids, params in zip(
pooled, prompt_token_ids, pooling_metadata.pooling_params
):
if hidden is None:
outputs.append(None)
continue
parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
if parsed is None:
outputs.append(hidden.new_empty(0, dtype=torch.float32))
continue
schema, lexical_ids, lexical_span_ids = self._indices(
*parsed, hidden.device
)
lexical = segment_mean(
self.embed_lexical(
async_tensor_h2d(token_ids[lexical_ids], hidden.device)
),
lexical_span_ids,
schema.span_lengths[len(parsed[0]) :],
)
logits = self.head(hidden, lexical, schema).float()
if params.use_activation is not False:
mask = schema.option_mask
grid = logits.new_full((mask.numel(),), float("-inf"))
grid[schema.option_slots] = logits
probs = grid.view(mask.shape).softmax(dim=-1)
logits = probs.view(-1)[schema.option_slots]
outputs.append(logits)
return outputs
@staticmethod
def _indices(
types: list[int],
spans: list[tuple[int, int]],
counts: list[int],
device: torch.device,
) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
"""The schema's indices, plus the option tokens' positions (on the CPU)
and their option spans, for the lexical embeddings."""
num_questions = len(types)
lengths = torch.tensor([end - start for start, end in spans])
span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
positions = torch.cat([torch.arange(start, end) for start, end in spans])
num_question_tokens = int(lengths[:num_questions].sum())
counts_cpu = torch.tensor(counts)
max_options = max(counts)
option_question = torch.repeat_interleave(
torch.arange(num_questions), counts_cpu
)
option_slots = torch.cat(
[torch.arange(count) + q * max_options for q, count in enumerate(counts)]
)
schema = SchemaIndices(
positions=async_tensor_h2d(positions, device),
span_ids=async_tensor_h2d(span_ids, device),
span_lengths=async_tensor_h2d(lengths, device),
question_types=async_tensor_h2d(types, device, dtype=torch.long),
option_question=async_tensor_h2d(option_question, device),
option_slots=async_tensor_h2d(option_slots, device),
option_mask=async_tensor_h2d(
torch.arange(max_options) < counts_cpu[:, None], device
),
)
lexical_span_ids = async_tensor_h2d(
span_ids[num_question_tokens:] - num_questions, device
)
return schema, positions[num_question_tokens:], lexical_span_ids
MAX_FRAME_PIXELS = 256 * 32 * 32
def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
"""The reference reads each video frame at at most 256 * 32 * 32 pixels,
in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on
their device (a CUDA tensor from torchcodec must not round-trip)."""
height, width = frames.shape[1:3]
if height * width <= MAX_FRAME_PIXELS:
return frames
scale = math.sqrt(MAX_FRAME_PIXELS / (width * height))
new_width = max(2, round(width * scale / 2) * 2)
new_height = max(2, round(height * scale / 2) * 2)
is_tensor = isinstance(frames, torch.Tensor)
tensor = frames if is_tensor else torch.from_numpy(frames)
resized = F.interpolate(
tensor.permute(0, 3, 1, 2).float(),
size=(new_height, new_width),
mode="bilinear",
)
if tensor.dtype == torch.uint8:
resized = resized.round()
resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous()
return resized if is_tensor else resized.numpy()
def _cap_video(item: Any) -> Any:
if isinstance(item, tuple):
frames, metadata = item
return cap_frame_pixels(frames), metadata
return cap_frame_pixels(item)
class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor):
"""Caps video frames at the reference's per-frame pixel count."""
def _call_hf_processor(
self,
hf_data: Mapping[str, object],
hf_kwargs: Mapping[str, object],
) -> BatchFeature:
if videos := hf_data.get("videos"):
assert isinstance(videos, list)
hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]}
return super()._call_hf_processor(hf_data, hf_kwargs)
@default_pooling_type(tok_pooling_type="ALL")
@MULTIMODAL_REGISTRY.register_processor(
ClefOmniMultiModalProcessor,
info=Qwen3OmniMoeThinkerProcessingInfo,
dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
)
class ClefOmniForJointSchema(Qwen3OmniMoeThinkerForConditionalGeneration):
is_pooling_model = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__(vllm_config=vllm_config, prefix=prefix)
model_config = vllm_config.model_config
head_config = model_config.hf_config.joint_head_config
self.pooler = JointSchemaPooler(
JointSchemaHead(**head_config), self._embed_lexical
)
# The head's weights ship as their own safetensors file.
self.secondary_weights = [
DefaultModelLoader.Source(
model_or_path=model_config.model,
revision=model_config.revision,
prefix="pooler.head.",
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
)
]
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
# The head embeds options with the output embeddings. ParallelLMHead
# forbids lookups, so use the vocab-parallel embedding's.
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
@@ -83,6 +83,7 @@ from vllm.multimodal.processing.processor import (
PromptUpdate,
PromptUpdateDetails,
)
from vllm.multimodal.utils import copy_mm_embedding_modality
from vllm.sequence import IntermediateTensors
from vllm.transformers_utils.processor import cached_processor_from_config
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
@@ -1498,8 +1499,12 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
audio_in_video_item_idx += 1
# Match the fps the processor used, as HF and the mrope positions do.
processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
if second_per_grid_ts:
if isinstance(processed_second_per_grid_ts, torch.Tensor):
video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item()
elif second_per_grid_ts:
video_second_per_grid_t = second_per_grid_ts[item_idx]
else:
video_second_per_grid_t = 2.0
@@ -1904,7 +1909,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
embeddings_main, embeddings_multiscale = torch.split(
embeddings, [visual_dim, multi_dim], dim=-1
)
multimodal_embeddings[index] = embeddings_main
multimodal_embeddings[index] = copy_mm_embedding_modality(
embeddings, embeddings_main
)
multimodal_embeddings_multiscale.append(embeddings_multiscale)
if not is_interleaved:
current_positions = mm_positions[
+1
View File
@@ -287,6 +287,7 @@ _REWARD_MODELS = {
_TOKEN_CLASSIFICATION_MODELS = {
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
"ModernBertForTokenClassification": (
"modernbert",
"ModernBertForTokenClassification",
+64
View File
@@ -1570,3 +1570,67 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
valid_frame_indices=valid_frame_indices,
)
return frames, metadata
@VIDEO_LOADER_REGISTRY.register("clef_omni")
class ClefOmniVideoBackend(VideoBackend):
"""Clef-Omni's reference sampling: the first frame at or after each
``1/fps`` boundary (frame i sits at ``i / original_fps`` seconds, so the
k-th boundary is crossed by frame ``ceil(k * original_fps / fps)``),
capped at 768 frames and padded to an even count by repeating the last
frame."""
MAX_FRAMES = 768
@classmethod
def compute_frames_index_to_sample(
cls,
source: VideoSourceMetadata,
target: VideoTargetMetadata,
**kwargs,
) -> list[int]:
if source.original_fps <= 0:
raise ValueError("video has an unknown frame rate")
step = source.original_fps / target.fps
indices = []
k = 0
while (index := math.ceil(k * step - 1e-6)) < source.total_frames_num:
indices.append(index)
if len(indices) > cls.MAX_FRAMES:
raise ValueError(
f"video is longer than {cls.MAX_FRAMES / target.fps:.0f} seconds"
)
k += 1
if not indices:
raise ValueError("video has no decodable frames")
return indices
@classmethod
def load_bytes(
cls,
data: bytes,
num_frames: int = -1,
fps: int = 2,
max_duration: int = 300,
frame_recovery: bool = False,
*,
backend: VideoDecoderBackend = "opencv",
**kwargs: Any,
) -> tuple[DecodedFrames, dict[str, Any]]:
frames, metadata = super().load_bytes(
data,
num_frames=num_frames,
fps=fps,
max_duration=max_duration,
frame_recovery=frame_recovery,
backend=backend,
**kwargs,
)
if len(frames) % 2:
last = frames[-1:]
if isinstance(frames, torch.Tensor):
frames = torch.cat([frames, last])
else:
frames = np.concatenate([frames, last])
metadata["frames_indices"].append(metadata["frames_indices"][-1])
return frames, metadata
@@ -17,6 +17,10 @@ from vllm.transformers_utils.config import (
ConfigFormat,
get_safetensors_params_metadata,
)
from vllm.transformers_utils.repo_utils import (
file_or_path_exists,
get_hf_file_to_dict,
)
from vllm.utils.torch_utils import common_broadcastable_dtype
logger = init_logger(__name__)
@@ -643,6 +647,31 @@ class MimoV2ModelArchConfigConvertor(ModelArchConfigConvertorBase):
_strip_mimo_v2_attention_chunk_size(hf_config, hf_text_config)
class Qwen3OmniMoeModelArchConfigConvertor(ModelArchConfigConvertorBase):
def __init__(
self,
hf_config: PreTrainedConfig,
hf_text_config: PreTrainedConfig,
revision: str | None = None,
):
# Clef-Omni keeps Qwen3-Omni's config and adds a joint schema head;
# route to its wrapper class when the head's config file is present.
# Mutate hf_config (like the deepseek_v4/mimo_v2 rewrites above)
# because model-class resolution reads hf_config.architectures.
if getattr(hf_config, "architectures", None) == [
"Qwen3OmniMoeForConditionalGeneration"
]:
model_path = hf_config.name_or_path
if model_path and file_or_path_exists(
model_path, "joint_head_config.json", revision
):
hf_config.architectures = ["ClefOmniForJointSchema"]
hf_config.joint_head_config = get_hf_file_to_dict(
"joint_head_config.json", model_path, revision
)
super().__init__(hf_config, hf_text_config, revision)
class MimoV2MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
def __init__(
self,
@@ -822,6 +851,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
"bailing_hybrid_v3_mtp": BailingHybridV3MTPModelArchConfigConvertor,
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
"qwen3_5_mtp": Qwen3_5MTPModelArchConfigConvertor,
"qwen3_omni_moe": Qwen3OmniMoeModelArchConfigConvertor,
"qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor,
"qwen4_exp_mtp": Qwen4ExpMTPModelArchConfigConvertor,
"RefinedWeb": FalconModelArchConfigConvertor,