[Model] Support Cloudflare/clef-omni on /v1/systemone (#61064)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Isotr0py
parent
314ab8cffd
commit
092c35dd15
@@ -130,6 +130,32 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a
|
||||
| options per `choice` | 26 |
|
||||
| levels per `score` | 10 |
|
||||
|
||||
## Clef-Omni
|
||||
|
||||
[Clef-Omni](https://huggingface.co/Cloudflare/clef-omni) is a decision model:
|
||||
a joint schema head on the Qwen3-Omni thinker scores every option of every
|
||||
question in one forward pass. vLLM serves it as a pooling model.
|
||||
|
||||
```bash
|
||||
vllm serve Cloudflare/clef-omni
|
||||
```
|
||||
|
||||
- A request is one read. The prompt is the one the checkpoint's
|
||||
`joint_schema_model.py` builds: the state, then every question with its
|
||||
options. A JSON state is written as compact JSON with sorted keys.
|
||||
- A state too long for the model is truncated to fit, as the reference does.
|
||||
A request whose questions alone exceed 64,000 tokens, or `max_model_len`,
|
||||
returns 400.
|
||||
- `instructions` returns 400, since the prompt is fixed. `chat_template_kwargs`
|
||||
has no effect.
|
||||
- There are no labels, so `label_mass` is 1 and `confidence` is the chosen
|
||||
option's probability. `usage.input_tokens` counts the prompt once, and
|
||||
`output_tokens` is 0.
|
||||
- `images`, `audio` and `videos` are read as the reference does: placed
|
||||
before the state, videos sampled at 2 fps, and a video's soundtrack heard
|
||||
with it when every video has one. Audio clips cannot be sent together with
|
||||
videos that have soundtracks.
|
||||
|
||||
## Request fields
|
||||
|
||||
| field | meaning |
|
||||
@@ -139,6 +165,7 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a
|
||||
| `instructions` | optional context placed ahead of the questions |
|
||||
| `chat_template_kwargs` | passed to the chat template, for example `{"enable_thinking": false}` |
|
||||
| `cache_salt` | optional [prefix-cache salt](../../usage/security.md#cache-salting) applied to every question |
|
||||
| `images`, `audio`, `videos` | optional media URLs or data URLs, for models that take them ([Clef-Omni](#clef-omni)) |
|
||||
|
||||
A question with any field other than `type`, `instructions` and `criteria` is
|
||||
rejected with a 400.
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import (
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.serving import (
|
||||
ServingStructuredDecisions,
|
||||
state_text,
|
||||
parse_questions,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
DiffusionGemmaCanvasStrategy,
|
||||
JointSchemaStrategy,
|
||||
NextTokenStrategy,
|
||||
ReadContext,
|
||||
reply_label_ids,
|
||||
select_read_strategy,
|
||||
state_text,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.register import (
|
||||
init_exception_handler,
|
||||
)
|
||||
from vllm.inputs import mm_input
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.multimodal.media import MediaConnector, MediaWithBytes
|
||||
from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
)
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
@@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
|
||||
|
||||
|
||||
def test_strategy_selection():
|
||||
qwen = "Qwen3ForCausalLM"
|
||||
assert select_read_strategy(model(qwen)) is NextTokenStrategy
|
||||
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
|
||||
qwen, generate = "Qwen3ForCausalLM", ("generate",)
|
||||
assert select_read_strategy(model(qwen), generate) is NextTokenStrategy
|
||||
assert (
|
||||
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"))
|
||||
select_read_strategy(model(qwen, "processed_logprobs"), generate)
|
||||
is NextTokenStrategy
|
||||
)
|
||||
assert (
|
||||
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate)
|
||||
is DiffusionGemmaCanvasStrategy
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
|
||||
select_read_strategy(model("LlamaForCausalLM"))
|
||||
select_read_strategy(model("LlamaForCausalLM"), generate)
|
||||
with pytest.raises(ValueError, match="not raw_logits"):
|
||||
select_read_strategy(model(qwen, "raw_logits"))
|
||||
select_read_strategy(model(qwen, "raw_logits"), generate)
|
||||
# A generative architecture served as a pooling model has no read strategy.
|
||||
with pytest.raises(ValueError, match="need the generate task"):
|
||||
select_read_strategy(model(qwen), ("embed",))
|
||||
# Clef-Omni scores options with its head, so it reads no logprobs.
|
||||
clef = model("ClefOmniForJointSchema", "raw_logits")
|
||||
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
|
||||
|
||||
|
||||
def test_route_is_registered_by_default():
|
||||
@@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serving_and_engine(qwen):
|
||||
def engine_and_renderer(qwen):
|
||||
tokenizer, _ = qwen
|
||||
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
|
||||
renderer = HfRenderer(
|
||||
@@ -148,6 +173,19 @@ def serving_and_engine(qwen):
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
yield engine, online_renderer
|
||||
renderer.shutdown()
|
||||
|
||||
|
||||
def serving(engine, strategy) -> ServingStructuredDecisions:
|
||||
name = engine.model_config.model
|
||||
models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)])
|
||||
return ServingStructuredDecisions(models, strategy)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serving_and_engine(engine_and_renderer):
|
||||
engine, online_renderer = engine_and_renderer
|
||||
|
||||
async def generate(prompt, params, request_id, **kwargs):
|
||||
label_ids = params.logprob_token_ids
|
||||
@@ -171,11 +209,7 @@ def serving_and_engine(qwen):
|
||||
|
||||
engine.generate.side_effect = generate
|
||||
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
|
||||
models = OpenAIServingModels(
|
||||
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
|
||||
)
|
||||
yield ServingStructuredDecisions(models, strategy), engine
|
||||
renderer.shutdown()
|
||||
return serving(engine, strategy), engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -256,3 +290,286 @@ def test_canvas_read():
|
||||
strategy._read_input(
|
||||
{"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4]
|
||||
)
|
||||
|
||||
|
||||
JOINT_SCHEMA_PROMPT = """<|im_start|>system
|
||||
Read the complete state and schema. Decide every field jointly. Each answer \
|
||||
must be exactly one of that field's allowed options.<|im_end|>
|
||||
<|im_start|>user
|
||||
STATE:
|
||||
{"customer":{"tier":"gold"},"ticket":"Charged twice"}
|
||||
|
||||
SCHEMA FIELDS:
|
||||
|
||||
FIELD 1
|
||||
ID: team
|
||||
TYPE: choice
|
||||
INSTRUCTION: Which team?
|
||||
ALLOWED OPTIONS:
|
||||
OPTION 1: {"description":"payments","option_id":"billing"}
|
||||
OPTION 2: {"option_id":"security"}
|
||||
OPTION 3: {"description":"deliveries","option_id":"shipping"}
|
||||
END FIELD
|
||||
|
||||
FIELD 2
|
||||
ID: urgent
|
||||
TYPE: noul
|
||||
INSTRUCTION: urgent
|
||||
ALLOWED OPTIONS:
|
||||
OPTION 1: {"description":"money is moving","option_id":"true"}
|
||||
OPTION 2: {"description":"The proposition is false or the answer is no.",\
|
||||
"option_id":"false"}
|
||||
END FIELD
|
||||
|
||||
FIELD 3
|
||||
ID: tone
|
||||
TYPE: score
|
||||
INSTRUCTION: How upset?
|
||||
ALLOWED OPTIONS:
|
||||
OPTION 1: {"description":"calm","option_id":"0"}
|
||||
OPTION 2: {"description":"annoyed","option_id":"1"}
|
||||
OPTION 3: {"description":"furious","option_id":"2"}
|
||||
END FIELD
|
||||
|
||||
<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
JOINT SCHEMA DECISIONS:"""
|
||||
|
||||
JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
|
||||
state={"ticket": "Charged twice", "customer": {"tier": "gold"}},
|
||||
questions={
|
||||
"team": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team?",
|
||||
"criteria": {
|
||||
"shipping": "deliveries",
|
||||
"billing": "payments",
|
||||
"security": None,
|
||||
},
|
||||
},
|
||||
"urgent": {"type": "noul", "criteria": {"true": "money is moving"}},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset?",
|
||||
"criteria": ["calm", "annoyed", "furious"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def joint_schema(qwen, engine_and_renderer):
|
||||
tokenizer, _ = qwen
|
||||
engine, online_renderer = engine_and_renderer
|
||||
# The probability the head gives each option, by option id.
|
||||
scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2}
|
||||
scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3}
|
||||
|
||||
async def encode(prompt, params, request_id, **kwargs):
|
||||
ids = prompt["prompt_token_ids"]
|
||||
spans = params.extra_kwargs["joint_schema"]["option_spans"]
|
||||
options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans]
|
||||
data = torch.tensor([scores[o["option_id"]] for o in options])
|
||||
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
|
||||
|
||||
engine.encode.side_effect = encode
|
||||
strategy = JointSchemaStrategy(
|
||||
ReadContext(engine, online_renderer, None, "auto", {})
|
||||
)
|
||||
return serving(engine, strategy), engine, tokenizer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_reads_every_question_at_once(joint_schema):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt")
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
|
||||
engine.encode.assert_called_once()
|
||||
engine.check_admission.assert_called_once_with(1)
|
||||
prompt, params = engine.encode.call_args.args[:2]
|
||||
ids = prompt["prompt_token_ids"]
|
||||
assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT
|
||||
assert prompt["cache_salt"] == "salt"
|
||||
assert params.task == "token_classify"
|
||||
schema = params.extra_kwargs["joint_schema"]
|
||||
assert schema["question_types"] == [1, 0, 2]
|
||||
assert schema["option_counts"] == [3, 2, 3]
|
||||
assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [
|
||||
"Which team?",
|
||||
"urgent",
|
||||
"How upset?",
|
||||
]
|
||||
|
||||
# Options are read in the model's order and answered in the request's.
|
||||
team = response.answers["team"]
|
||||
assert team["choice"] == "billing"
|
||||
assert list(team["probabilities"]) == ["shipping", "billing", "security"]
|
||||
assert team["probabilities"] == pytest.approx(
|
||||
{"shipping": 0.2, "billing": 0.7, "security": 0.1}
|
||||
)
|
||||
assert response.answers["urgent"]["noul"] == pytest.approx(0.9)
|
||||
assert response.answers["tone"]["score"] == pytest.approx(1.1)
|
||||
assert response.usage.input_tokens == len(ids)
|
||||
assert response.usage.output_tokens == 0
|
||||
assert all(d.label_mass == 1.0 for d in response.diagnostics.values())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_rejects_instructions(joint_schema):
|
||||
serving, engine, _ = joint_schema
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.")
|
||||
)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
engine.encode.assert_not_called()
|
||||
|
||||
|
||||
def test_joint_schema_truncates_the_state(joint_schema):
|
||||
serving, _, _ = joint_schema
|
||||
strategy = serving.strategy
|
||||
request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST)
|
||||
questions = parse_questions(request, strategy.limits())
|
||||
prompt, schema, _ = strategy.encode(questions, "word " * 100)
|
||||
strategy.max_length = len(prompt) - 50
|
||||
truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100)
|
||||
# The state loses 50 tokens and the questions stay at the end.
|
||||
assert len(truncated) == strategy.max_length
|
||||
assert truncated[-200:] == prompt[-200:]
|
||||
assert truncated_schema == schema
|
||||
strategy.max_length = 100
|
||||
with pytest.raises(StructuredDecisionError, match="questions take"):
|
||||
strategy.encode(questions, "word")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_needs_a_model_that_takes_it(serving_and_engine):
|
||||
serving, engine = serving_and_engine
|
||||
request = StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"]
|
||||
)
|
||||
response = await serving.create_decision(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert "does not take image inputs" in response.error.message
|
||||
engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
strategy = serving.strategy
|
||||
strategy.__dict__["_connector"] = MediaConnector()
|
||||
image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
|
||||
rendered: dict[str, Any] = {}
|
||||
|
||||
async def process_for_engine_async(prompt, arrival_time):
|
||||
rendered.update(prompt)
|
||||
# The image's placeholder expands to three tokens.
|
||||
ids = [
|
||||
token
|
||||
for pad in prompt["prompt_token_ids"]
|
||||
for token in ([pad] * 3 if pad == image_pad else [pad])
|
||||
]
|
||||
return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt"))
|
||||
|
||||
renderer = strategy.context.online_renderer.renderer
|
||||
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
|
||||
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
|
||||
placeholder = tokenizer.encode(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
|
||||
)
|
||||
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
|
||||
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE}
|
||||
}
|
||||
prompt = engine.encode.call_args.args[0]
|
||||
assert prompt["type"] == "multimodal"
|
||||
ids = prompt["prompt_token_ids"]
|
||||
expanded = len(strategy.prefix) + len(placeholder) + 2
|
||||
assert ids[:expanded].count(image_pad) == 3
|
||||
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
|
||||
assert tokenizer.decode(ids[expanded:]) == state_and_questions
|
||||
assert response.answers["team"]["choice"] == "billing"
|
||||
assert response.usage.input_tokens == len(ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
strategy = serving.strategy
|
||||
video = MediaWithBytes(
|
||||
(np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes"
|
||||
)
|
||||
track = (np.zeros(160, np.float32), 16000)
|
||||
rendered: dict[str, Any] = {}
|
||||
|
||||
class FakeConnector:
|
||||
async def fetch_image_async(self, url):
|
||||
raise AssertionError("no images in this test")
|
||||
|
||||
async def load_from_url_async(self, url, media_io, *, fetch_timeout=None):
|
||||
return video
|
||||
|
||||
async def fetch_audio_async(self, url):
|
||||
if url == "https://example.org/silent.mp4":
|
||||
raise ValueError("No audio stream found.")
|
||||
return track
|
||||
|
||||
strategy.__dict__["_connector"] = FakeConnector()
|
||||
|
||||
async def process_for_engine_async(prompt, arrival_time):
|
||||
rendered.clear()
|
||||
rendered.update(prompt)
|
||||
return mm_input(prompt["prompt_token_ids"], {}, {}, {})
|
||||
|
||||
renderer = strategy.context.online_renderer.renderer
|
||||
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
|
||||
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"]
|
||||
)
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE},
|
||||
"use_audio_in_video": True,
|
||||
}
|
||||
|
||||
# A video without a decodable soundtrack is watched silently.
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"]
|
||||
)
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert rendered["multi_modal_data"] == {"video": [video]}
|
||||
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
|
||||
|
||||
# Audio clips cannot be read with videos that have soundtracks.
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST,
|
||||
videos=["https://example.org/talking.mp4"],
|
||||
audio=["https://example.org/clip.wav"],
|
||||
)
|
||||
)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
|
||||
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
|
||||
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
|
||||
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
|
||||
]
|
||||
|
||||
|
||||
def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
|
||||
"""A source with explicit allow patterns (e.g. a standalone head file in
|
||||
the repo root) must not be validated against the backbone's index."""
|
||||
save_file(
|
||||
{"backbone.weight": torch.zeros(1)},
|
||||
tmp_path / "model-00001-of-00001.safetensors",
|
||||
)
|
||||
save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
|
||||
(tmp_path / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
|
||||
)
|
||||
)
|
||||
|
||||
loader = DefaultModelLoader(LoadConfig())
|
||||
secondary = DefaultModelLoader.Source(
|
||||
model_or_path=str(tmp_path),
|
||||
revision=None,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=["joint_head.safetensors"],
|
||||
)
|
||||
assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
|
||||
"pooler.head.head.weight"
|
||||
]
|
||||
# The primary source still filters files the backbone index doesn't list.
|
||||
primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
|
||||
assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
|
||||
which loops over questions where the pooler batches them."""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
HIDDEN, VOCAB = 64, 500
|
||||
TYPES = [0, 1, 1, 2]
|
||||
COUNTS = [2, 3, 1, 5]
|
||||
|
||||
|
||||
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
|
||||
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
|
||||
for one record, with ``spans`` the option spans of each question."""
|
||||
hidden = head.hidden_norm(hidden)
|
||||
memory = head.memory_projection(hidden).unsqueeze(0)
|
||||
global_vector = hidden[-1]
|
||||
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
|
||||
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
|
||||
lexicals = [
|
||||
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
|
||||
]
|
||||
options = torch.cat(
|
||||
[
|
||||
head.option_context_projection(c)
|
||||
+ head.option_lexical_projection(lex)
|
||||
+ head.option_question_projection(questions[i]).unsqueeze(0)
|
||||
for i, (c, lex) in enumerate(zip(contexts, lexicals))
|
||||
]
|
||||
).unsqueeze(0)
|
||||
for layer in head.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
routed = torch.split(options[0], [len(q) for q in spans])
|
||||
|
||||
base = head.question_projection(questions)
|
||||
summaries = []
|
||||
for field, opts in zip(base, routed):
|
||||
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
|
||||
summaries.append((weights[:, None] * opts).sum(0))
|
||||
fields = (
|
||||
base
|
||||
+ head.option_summary_norm(torch.stack(summaries))
|
||||
+ head.global_projection(global_vector)
|
||||
+ head.type_embedding(torch.tensor(TYPES))
|
||||
).unsqueeze(0)
|
||||
for layer in head.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = head.field_norm(fields[0])
|
||||
|
||||
logits = []
|
||||
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
|
||||
anchor = F.normalize(questions[i] + global_vector, dim=-1)
|
||||
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
|
||||
opts = head.option_norm(opts)
|
||||
field = field.expand_as(opts)
|
||||
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
|
||||
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
|
||||
joint = joint + head.residual_scorer(features).squeeze(-1)
|
||||
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
|
||||
return logits
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pooler():
|
||||
torch.manual_seed(0)
|
||||
head = JointSchemaHead(
|
||||
hidden_size=HIDDEN,
|
||||
width=32,
|
||||
routing_layers=2,
|
||||
layers=2,
|
||||
heads=4,
|
||||
feedforward=48,
|
||||
).eval()
|
||||
with torch.no_grad():
|
||||
for p in head.parameters():
|
||||
p.add_(torch.randn_like(p) * 0.2)
|
||||
embedding = torch.randn(VOCAB, HIDDEN)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(
|
||||
enable_chunked_prefill=False, async_scheduling=False
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=config,
|
||||
):
|
||||
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
|
||||
return pooler, embedding
|
||||
|
||||
|
||||
def schema(length: int):
|
||||
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
|
||||
pos = 20
|
||||
question_spans: list[tuple[int, int]] = []
|
||||
spans: list[list[tuple[int, int]]] = []
|
||||
for count in COUNTS:
|
||||
question_spans.append((pos, pos + 4))
|
||||
pos += 6
|
||||
spans.append([])
|
||||
for k in range(count):
|
||||
n = 1 + (k * 3) % 4
|
||||
spans[-1].append((pos, pos + n))
|
||||
pos += n + 1
|
||||
assert pos < length
|
||||
extra = {
|
||||
"question_types": TYPES,
|
||||
"question_spans": [[s - length, e - length] for s, e in question_spans],
|
||||
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
|
||||
"option_counts": COUNTS,
|
||||
}
|
||||
return {"joint_schema": extra}, question_spans, spans
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_pooler_matches_reference(pooler):
|
||||
pooler, embedding = pooler
|
||||
lengths = [120, 90, 100]
|
||||
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
|
||||
hidden = torch.randn(sum(lengths), HIDDEN)
|
||||
first, first_questions, first_spans = schema(lengths[0])
|
||||
last, last_questions, last_spans = schema(lengths[2])
|
||||
params = [
|
||||
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
|
||||
# A prompt without a schema, like the profiling run's.
|
||||
PoolingParams(task="token_classify", use_activation=True),
|
||||
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
|
||||
]
|
||||
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
|
||||
for i, ids in enumerate(token_ids):
|
||||
padded[i, : len(ids)] = ids
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor(lengths),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=padded,
|
||||
pooling_params=params,
|
||||
pooling_states=[PoolingStates() for _ in lengths],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
torch.tensor(lengths).numpy(),
|
||||
seq_lens_cpu=torch.tensor(lengths),
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
probs, empty, logits = pooler(hidden, metadata)
|
||||
|
||||
first_hidden, _, last_hidden = torch.split(hidden, lengths)
|
||||
expected = reference_logits(
|
||||
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
|
||||
)
|
||||
expected_probs = torch.cat([q.softmax(-1) for q in expected])
|
||||
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
|
||||
assert empty.numel() == 0
|
||||
expected = reference_logits(
|
||||
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
|
||||
)
|
||||
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
|
||||
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
|
||||
f"Got short={short_tokens}, long={long_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
|
||||
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
|
||||
"""Audio and video tokens interleave by the fps the video was processed
|
||||
at, as in HF, not by the default of one frame a second."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
rng = np.random.RandomState(42)
|
||||
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
|
||||
audio = rng.rand(32000).astype(np.float32)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
"use_audio_in_video": True,
|
||||
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [video], "audio": [(audio, 16000)]}
|
||||
),
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
)
|
||||
expected = processor.info.get_hf_processor()(
|
||||
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
|
||||
)
|
||||
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
@@ -1351,6 +1351,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"ClefOmniForJointSchema": _HfExamplesInfo(
|
||||
"Cloudflare/clef-omni",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
ClefOmniVideoBackend,
|
||||
DynamicVideoBackend,
|
||||
Glm5NextVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
@@ -1813,3 +1814,18 @@ def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
# One initial seek, then pure walking -- no re-seek churn.
|
||||
assert cap.seeks == 1
|
||||
|
||||
|
||||
def test_clef_omni_backend_samples_reference_frames(tmp_path):
|
||||
"""Clef-Omni samples the first frame at or after each 0.5 s boundary
|
||||
(the reference's 2 fps timestamp sampling), not uniform strides."""
|
||||
pytest.importorskip("cv2")
|
||||
|
||||
# 2 s at 30 fps: the reference reads frames 0, 15, 30 and 45, where
|
||||
# uniform sampling would read 0, 19, 39 and 59.
|
||||
path = _write_gray_video(tmp_path, total_frames=60, fps=30)
|
||||
frames, metadata = ClefOmniVideoBackend.load_bytes(path.read_bytes())
|
||||
assert metadata["frames_indices"] == [0, 15, 30, 45]
|
||||
assert metadata["do_sample_frames"] is False
|
||||
for frame, idx in zip(frames, [0, 15, 30, 45]):
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
|
||||
@@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any
|
||||
from fastapi import FastAPI
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.logger import init_logger
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
@@ -18,8 +17,6 @@ if TYPE_CHECKING:
|
||||
else:
|
||||
RequestLogger = object
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def register_generate_api_routers(app: FastAPI):
|
||||
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||
@@ -56,12 +53,6 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_generative_scoring_api_router(app)
|
||||
|
||||
from .structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
|
||||
async def init_generate_state(
|
||||
engine_client: "EngineClient",
|
||||
@@ -231,33 +222,3 @@ async def init_generate_state(
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
from .structured_decisions.serving import ServingStructuredDecisions
|
||||
from .structured_decisions.strategies import ReadContext, select_read_strategy
|
||||
|
||||
strategy = None
|
||||
if "generate" in supported_tasks:
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
except ValueError as e:
|
||||
# Info, since every model without a read strategy lands here.
|
||||
logger.info("/v1/systemone is disabled: %s", e)
|
||||
strategy = None
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
if strategy is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -1,19 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
from .strategies import ReadContext, select_read_strategy
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -50,3 +61,31 @@ async def create_structured_decision(
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def init_structured_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
state.serving_structured_decisions = None
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=load_chat_template(args.chat_template),
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.debug("/v1/systemone is disabled: %s", e)
|
||||
return
|
||||
state.serving_structured_decisions = ServingStructuredDecisions(
|
||||
state.openai_serving_models, strategy, request_logger=request_logger
|
||||
)
|
||||
|
||||
@@ -30,6 +30,15 @@ class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
instructions: str | None = Field(
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
images: list[str] | None = Field(
|
||||
default=None, description="Image URLs or data URLs, for models that see."
|
||||
)
|
||||
audio: list[str] | None = Field(
|
||||
default=None, description="Audio URLs or data URLs, for models that hear."
|
||||
)
|
||||
videos: list[str] | None = Field(
|
||||
default=None, description="Video URLs or data URLs, for models that see."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
@@ -24,10 +23,6 @@ from .strategies import DecisionLimits, ReadStrategy
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def state_text(state: Any) -> str:
|
||||
return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
request: StructuredDecisionRequest,
|
||||
limits: DecisionLimits,
|
||||
@@ -72,6 +67,18 @@ class ServingStructuredDecisions(BaseServing):
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
|
||||
urls = {
|
||||
"image": request.images,
|
||||
"audio": request.audio,
|
||||
"video": request.videos,
|
||||
}
|
||||
media = {modality: items for modality, items in urls.items() if items}
|
||||
if unsupported := media.keys() - self.strategy.modalities:
|
||||
names = ", ".join(sorted(unsupported))
|
||||
raise StructuredDecisionError(f"this model does not take {names} inputs")
|
||||
return media
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
@@ -87,8 +94,9 @@ class ServingStructuredDecisions(BaseServing):
|
||||
request_id = f"decision-{base_id}"
|
||||
try:
|
||||
questions = parse_questions(request, self.limits)
|
||||
media = self._media(request)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(len(questions))
|
||||
engine_client.check_admission(self.strategy.num_requests(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
@@ -97,13 +105,14 @@ class ServingStructuredDecisions(BaseServing):
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
state_text(request.state),
|
||||
self.strategy.render_state(request.state),
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
media=media,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
|
||||
@@ -5,24 +5,40 @@
|
||||
Only models in READ_STRATEGIES are currently supported.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
from functools import cached_property
|
||||
from typing import Any, ClassVar, TypeVar
|
||||
|
||||
import numpy as np
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.label_reads import next_token_label_reads
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.multimodal.media import (
|
||||
MEDIA_CONNECTOR_REGISTRY,
|
||||
ImageMediaIO,
|
||||
MediaConnector,
|
||||
MediaIO,
|
||||
VideoMediaIO,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .protocol import ReadPromptRequest
|
||||
@@ -34,6 +50,10 @@ from .question_types import (
|
||||
label_softmax,
|
||||
)
|
||||
|
||||
_M = TypeVar("_M")
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReadContext:
|
||||
@@ -59,13 +79,30 @@ class QuestionRead:
|
||||
output_tokens: int
|
||||
|
||||
|
||||
def state_text(state: Any) -> str:
|
||||
return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False)
|
||||
|
||||
|
||||
class ReadStrategy(ABC):
|
||||
#: The engine task reads run as.
|
||||
task: ClassVar[SupportedTask] = "generate"
|
||||
#: The modalities a request may attach media of.
|
||||
modalities: ClassVar[frozenset[str]] = frozenset()
|
||||
|
||||
def __init__(self, context: ReadContext):
|
||||
self.context = context
|
||||
|
||||
@abstractmethod
|
||||
def limits(self) -> DecisionLimits: ...
|
||||
|
||||
def render_state(self, state: Any) -> str:
|
||||
"""The request's state as the model reads it."""
|
||||
return state_text(state)
|
||||
|
||||
def num_requests(self, questions: list[Question]) -> int:
|
||||
"""The engine requests a decision takes."""
|
||||
return len(questions)
|
||||
|
||||
@abstractmethod
|
||||
async def read(
|
||||
self,
|
||||
@@ -79,6 +116,7 @@ class ReadStrategy(ABC):
|
||||
priority: int,
|
||||
cache_salt: str | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
media: Mapping[str, Sequence[str]] | None = None,
|
||||
) -> list[QuestionRead]:
|
||||
"""One read per question, in order."""
|
||||
|
||||
@@ -214,6 +252,7 @@ class NextTokenStrategy(ReadStrategy):
|
||||
priority: int,
|
||||
cache_salt: str | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
media: Mapping[str, Sequence[str]] | None = None,
|
||||
) -> list[QuestionRead]:
|
||||
ctx = self.context
|
||||
read_request = self._read_request(chat_template_kwargs, cache_salt)
|
||||
@@ -334,7 +373,315 @@ class DiffusionGemmaCanvasStrategy(NextTokenStrategy):
|
||||
)
|
||||
|
||||
|
||||
class JointSchemaStrategy(ReadStrategy):
|
||||
"""Clef-Omni. The whole request is one pooling read: the state, then every
|
||||
question with its options, as the model was trained to read them. The
|
||||
model's joint schema head scores each option, so a read has no labels.
|
||||
|
||||
Mirrors ``encode_record`` in the checkpoint's ``joint_schema_model.py``.
|
||||
Each piece is tokenized on its own, so the spans of the questions and the
|
||||
options are known. The spans count from the end of the prompt, so they
|
||||
hold after the media placeholders, which come before the state, expand."""
|
||||
|
||||
task = "token_classify"
|
||||
modalities = frozenset({"image", "audio", "video"})
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"Read the complete state and schema. Decide every field jointly. Each "
|
||||
"answer must be exactly one of that field's allowed options."
|
||||
)
|
||||
#: The reference's default bound on a prompt, in tokens.
|
||||
MAX_LENGTH = 64000
|
||||
TYPE_IDS = {"noul": 0, "choice": 1, "score": 2}
|
||||
NOUL_CRITERIA = {
|
||||
"true": "The proposition is true or the answer is yes.",
|
||||
"false": "The proposition is false or the answer is no.",
|
||||
}
|
||||
PLACEHOLDERS = {
|
||||
"image": "<|vision_start|><|image_pad|><|vision_end|>\n",
|
||||
"audio": "<|audio_start|><|audio_pad|><|audio_end|>\n",
|
||||
"video": "<|vision_start|><|video_pad|><|vision_end|>\n",
|
||||
}
|
||||
IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_048_576}
|
||||
#: The reference samples videos at 2 fps; the "clef_omni" video backend
|
||||
#: reproduces the reference's sampling, pixel cap and frame count cap.
|
||||
VIDEO_FPS = 2
|
||||
|
||||
def __init__(self, context: ReadContext):
|
||||
super().__init__(context)
|
||||
self.tokenizer = context.online_renderer.renderer.get_tokenizer()
|
||||
self.max_length = min(
|
||||
self.MAX_LENGTH, context.engine_client.model_config.max_model_len
|
||||
)
|
||||
self.prefix = self._tokens(
|
||||
f"<|im_start|>system\n{self.SYSTEM_PROMPT}<|im_end|>\n"
|
||||
"<|im_start|>user\nSTATE:\n"
|
||||
)
|
||||
self.suffix = self._tokens(
|
||||
"\n<|im_end|>\n<|im_start|>assistant\n<think>\n\n</think>\n\n"
|
||||
"JOINT SCHEMA DECISIONS:"
|
||||
)
|
||||
|
||||
def limits(self) -> DecisionLimits:
|
||||
return DecisionLimits(max_questions=64, max_options=len(LABELS))
|
||||
|
||||
def render_state(self, state: Any) -> str:
|
||||
return self._render(state)
|
||||
|
||||
def num_requests(self, questions: list[Question]) -> int:
|
||||
return 1
|
||||
|
||||
@staticmethod
|
||||
def _render(value: Any) -> str:
|
||||
if isinstance(value, str):
|
||||
return value
|
||||
return json.dumps(
|
||||
value, ensure_ascii=False, separators=(",", ":"), sort_keys=True
|
||||
)
|
||||
|
||||
def _tokens(self, text: str) -> list[int]:
|
||||
return self.tokenizer.encode(text, add_special_tokens=False)
|
||||
|
||||
@cached_property
|
||||
def _connector(self) -> MediaConnector:
|
||||
model_config = self.context.engine_client.model_config
|
||||
return MEDIA_CONNECTOR_REGISTRY.load(
|
||||
envs.VLLM_MEDIA_CONNECTOR,
|
||||
media_io_kwargs=model_config.get_multimodal_config().media_io_kwargs,
|
||||
allowed_local_media_path=model_config.allowed_local_media_path,
|
||||
allowed_media_domains=model_config.allowed_media_domains,
|
||||
)
|
||||
|
||||
async def _fetch(
|
||||
self, urls: Sequence[str], media_io: MediaIO[_M], timeout: int
|
||||
) -> list[_M]:
|
||||
return await asyncio.gather(
|
||||
*(
|
||||
self._connector.load_from_url_async(
|
||||
url, media_io, fetch_timeout=timeout
|
||||
)
|
||||
for url in urls
|
||||
)
|
||||
)
|
||||
|
||||
async def _soundtrack(
|
||||
self, video_url: str
|
||||
) -> tuple[np.ndarray, int | float] | None:
|
||||
"""The video's soundtrack, or None when it has no decodable one."""
|
||||
try:
|
||||
return await self._connector.fetch_audio_async(video_url)
|
||||
except ValueError as e:
|
||||
logger.debug("no decodable soundtrack in %s: %s", video_url, e)
|
||||
return None
|
||||
|
||||
async def _render_media(
|
||||
self, media: Mapping[str, Sequence[str]], cache_salt: str | None
|
||||
) -> MultiModalInput:
|
||||
"""The system prompt and the media, with the media's placeholders
|
||||
expanded. A video's soundtrack is heard with it when every video has
|
||||
one."""
|
||||
connector = self._connector
|
||||
video_io = VideoMediaIO(
|
||||
ImageMediaIO(),
|
||||
num_frames=-1,
|
||||
fps=self.VIDEO_FPS,
|
||||
video_backend="clef_omni",
|
||||
)
|
||||
try:
|
||||
images, audio, videos, tracks = await asyncio.gather(
|
||||
asyncio.gather(
|
||||
*map(connector.fetch_image_async, media.get("image", ()))
|
||||
),
|
||||
asyncio.gather(
|
||||
*map(connector.fetch_audio_async, media.get("audio", ()))
|
||||
),
|
||||
self._fetch(
|
||||
media.get("video", ()), video_io, envs.VLLM_VIDEO_FETCH_TIMEOUT
|
||||
),
|
||||
asyncio.gather(*map(self._soundtrack, media.get("video", ()))),
|
||||
)
|
||||
except ValueError as e:
|
||||
raise StructuredDecisionError(f"media: {e}") from e
|
||||
|
||||
audio_in_video = bool(videos) and all(t is not None for t in tracks)
|
||||
if audio_in_video and audio:
|
||||
raise StructuredDecisionError(
|
||||
"audio clips cannot be read with videos that have soundtracks"
|
||||
)
|
||||
text = (
|
||||
self.PLACEHOLDERS["image"] * len(images)
|
||||
+ self.PLACEHOLDERS["audio"] * len(audio)
|
||||
+ self.PLACEHOLDERS["video"] * len(videos)
|
||||
)
|
||||
if audio_in_video:
|
||||
audio = [track for track in tracks if track is not None]
|
||||
mm_data: dict[str, Any] = {}
|
||||
if images:
|
||||
mm_data["image"] = images
|
||||
if audio:
|
||||
mm_data["audio"] = audio
|
||||
if videos:
|
||||
mm_data["video"] = videos
|
||||
mm_kwargs: dict[str, Any] = {"images_kwargs": {"size": self.IMAGE_SIZE}}
|
||||
if videos:
|
||||
mm_kwargs["use_audio_in_video"] = audio_in_video
|
||||
prompt = TokensPrompt(
|
||||
prompt_token_ids=self.prefix + self._tokens(text),
|
||||
multi_modal_data=mm_data,
|
||||
mm_processor_kwargs=mm_kwargs,
|
||||
)
|
||||
if cache_salt is not None:
|
||||
prompt["cache_salt"] = cache_salt
|
||||
renderer = self.context.online_renderer.renderer
|
||||
try:
|
||||
engine_input = await renderer.process_for_engine_async(prompt, time.time())
|
||||
except ValueError as e:
|
||||
raise StructuredDecisionError(f"media: {e}") from e
|
||||
assert engine_input["type"] == "multimodal"
|
||||
return engine_input
|
||||
|
||||
def _options(self, q: Question) -> list[tuple[int, str, Any]]:
|
||||
"""Each option's index in ``q.options``, id and description, in the
|
||||
model's order: choices sorted by name, noul true then false, score
|
||||
levels by index."""
|
||||
if q.type.name == "noul":
|
||||
return [
|
||||
(
|
||||
i,
|
||||
key,
|
||||
self.NOUL_CRITERIA[key] if o.description is None else o.description,
|
||||
)
|
||||
for i, (key, o) in enumerate(zip(("true", "false"), q.options))
|
||||
]
|
||||
if q.type.name == "choice":
|
||||
return sorted(
|
||||
((i, o.name, o.description) for i, o in enumerate(q.options)),
|
||||
key=lambda option: option[1],
|
||||
)
|
||||
if q.type.name == "score":
|
||||
return [(i, str(i), o.name) for i, o in enumerate(q.options)]
|
||||
raise StructuredDecisionError(
|
||||
f"question {q.id!r}: this model does not answer {q.type.name!r} questions"
|
||||
)
|
||||
|
||||
def encode(
|
||||
self, questions: list[Question], state: str, prefix: list[int] | None = None
|
||||
) -> tuple[list[int], dict[str, list[Any]], list[list[int]]]:
|
||||
"""The prompt, the schema for the pooler, and each question's option
|
||||
indices in the order of the output. ``prefix`` is what precedes the
|
||||
state: the system prompt, and the media if there are any."""
|
||||
prefix = self.prefix if prefix is None else prefix
|
||||
schema_ids = self._tokens("\n\nSCHEMA FIELDS:\n")
|
||||
question_spans, option_spans, orders = [], [], []
|
||||
for n, q in enumerate(questions, start=1):
|
||||
schema_ids += self._tokens(
|
||||
f"\nFIELD {n}\nID: {q.id}\nTYPE: {q.type.name}\nINSTRUCTION: "
|
||||
)
|
||||
start = len(schema_ids)
|
||||
schema_ids += self._tokens(q.instructions or q.id)
|
||||
question_spans.append([start, len(schema_ids)])
|
||||
schema_ids += self._tokens("\nALLOWED OPTIONS:\n")
|
||||
options = self._options(q)
|
||||
for m, (_, option_id, description) in enumerate(options, start=1):
|
||||
schema_ids += self._tokens(f"OPTION {m}: ")
|
||||
start = len(schema_ids)
|
||||
semantics = {"option_id": option_id}
|
||||
if description is not None:
|
||||
semantics["description"] = description
|
||||
schema_ids += self._tokens(self._render(semantics))
|
||||
option_spans.append([start, len(schema_ids)])
|
||||
schema_ids += self._tokens("\n")
|
||||
schema_ids += self._tokens("END FIELD\n")
|
||||
orders.append([i for i, _, _ in options])
|
||||
|
||||
fixed = len(prefix) + len(schema_ids) + len(self.suffix)
|
||||
if fixed > self.max_length:
|
||||
raise StructuredDecisionError(
|
||||
f"the questions take {fixed} tokens, more than the "
|
||||
f"{self.max_length} this model reads"
|
||||
)
|
||||
state_ids = self._tokens(state)[: self.max_length - fixed]
|
||||
prompt = prefix + state_ids + schema_ids + self.suffix
|
||||
# The schema ends len(self.suffix) tokens before the prompt does.
|
||||
end = len(schema_ids) + len(self.suffix)
|
||||
schema: dict[str, list[Any]] = {
|
||||
"question_types": [self.TYPE_IDS[q.type.name] for q in questions],
|
||||
"question_spans": [[s - end, e - end] for s, e in question_spans],
|
||||
"option_spans": [[s - end, e - end] for s, e in option_spans],
|
||||
"option_counts": [len(order) for order in orders],
|
||||
}
|
||||
return prompt, schema, orders
|
||||
|
||||
async def read(
|
||||
self,
|
||||
questions: list[Question],
|
||||
instructions: str | None,
|
||||
state: str,
|
||||
*,
|
||||
request_id: str,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
cache_salt: str | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
media: Mapping[str, Sequence[str]] | None = None,
|
||||
) -> list[QuestionRead]:
|
||||
if instructions:
|
||||
raise StructuredDecisionError(
|
||||
"this model reads a fixed prompt, so it takes no instructions"
|
||||
)
|
||||
engine_input: EngineInput
|
||||
if media:
|
||||
engine_input = await self._render_media(media, cache_salt)
|
||||
prompt, schema, orders = self.encode(
|
||||
questions, state, engine_input["prompt_token_ids"]
|
||||
)
|
||||
engine_input["prompt_token_ids"] = prompt
|
||||
else:
|
||||
prompt, schema, orders = self.encode(questions, state)
|
||||
engine_input = tokens_input(prompt, cache_salt=cache_salt)
|
||||
result = None
|
||||
async for result in self.context.engine_client.encode(
|
||||
engine_input,
|
||||
PoolingParams(task="token_classify", extra_kwargs={"joint_schema": schema}),
|
||||
f"{request_id}-0",
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
):
|
||||
pass
|
||||
if result is None:
|
||||
raise ValueError("Failed to score the questions")
|
||||
if result.error is not None:
|
||||
raise ValueError(f"Failed to score the questions: {result.error.message}")
|
||||
probs = result.outputs.data.tolist()
|
||||
if len(probs) != sum(schema["option_counts"]):
|
||||
raise ValueError(
|
||||
f"Expected {sum(schema['option_counts'])} option scores, "
|
||||
f"got {len(probs)}"
|
||||
)
|
||||
|
||||
reads, offset = [], 0
|
||||
for n, order in enumerate(orders):
|
||||
q_probs = [0.0] * len(order)
|
||||
for i, p in zip(order, probs[offset : offset + len(order)]):
|
||||
q_probs[i] = p
|
||||
offset += len(order)
|
||||
reads.append(
|
||||
QuestionRead(
|
||||
probs=q_probs,
|
||||
label_mass=1.0,
|
||||
argmax_is_label=True,
|
||||
# One forward pass reads the prompt for every question.
|
||||
input_tokens=len(prompt) if n == 0 else 0,
|
||||
output_tokens=0,
|
||||
)
|
||||
)
|
||||
return reads
|
||||
|
||||
|
||||
READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
|
||||
"ClefOmniForJointSchema": JointSchemaStrategy,
|
||||
"Qwen3ForCausalLM": NextTokenStrategy,
|
||||
"Qwen3_5ForConditionalGeneration": NextTokenStrategy,
|
||||
"Qwen3_5MoeForConditionalGeneration": NextTokenStrategy,
|
||||
@@ -347,7 +694,9 @@ READ_STRATEGIES: dict[str, type[ReadStrategy]] = {
|
||||
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
|
||||
|
||||
|
||||
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
|
||||
def select_read_strategy(
|
||||
model_config: ModelConfig, supported_tasks: Sequence[SupportedTask]
|
||||
) -> type[ReadStrategy]:
|
||||
"""Raise ValueError when the model cannot serve structured decisions."""
|
||||
strategy = READ_STRATEGIES.get(model_config.architecture)
|
||||
if strategy is None:
|
||||
@@ -356,7 +705,12 @@ def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
|
||||
f"{model_config.architecture}. Supported architectures: "
|
||||
f"{sorted(READ_STRATEGIES)}"
|
||||
)
|
||||
if model_config.logprobs_mode not in LOGPROBS_MODES:
|
||||
if strategy.task not in supported_tasks:
|
||||
raise ValueError(
|
||||
f"Structured decisions on {model_config.architecture} need the "
|
||||
f"{strategy.task} task"
|
||||
)
|
||||
if strategy.task == "generate" and model_config.logprobs_mode not in LOGPROBS_MODES:
|
||||
raise ValueError(
|
||||
"Structured decisions need --logprobs-mode raw_logprobs or "
|
||||
f"processed_logprobs, not {model_config.logprobs_mode}"
|
||||
|
||||
@@ -157,6 +157,19 @@ async def init_app_state(
|
||||
|
||||
init_pooling_state(engine_client, state, args, request_logger, supported_tasks)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
init_structured_decisions_state,
|
||||
)
|
||||
|
||||
init_structured_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
supported_tasks,
|
||||
default_chat_template_kwargs,
|
||||
)
|
||||
|
||||
await init_endpoint_plugins_state(engine_client, state, args)
|
||||
|
||||
state.enable_server_load_tracking = args.enable_server_load_tracking
|
||||
|
||||
@@ -25,6 +25,12 @@ def register_api_routers(
|
||||
|
||||
register_models_api_router(app)
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
@@ -247,9 +247,11 @@ class DefaultModelLoader(BaseModelLoader):
|
||||
subfolder=subfolder,
|
||||
revision=revision,
|
||||
)
|
||||
hf_weights_files = filter_duplicate_safetensors_files(
|
||||
hf_weights_files, hf_folder, index_file
|
||||
)
|
||||
# only filter the weights from index without explicit allow patterns
|
||||
if allow_patterns_overrides is None:
|
||||
hf_weights_files = filter_duplicate_safetensors_files(
|
||||
hf_weights_files, hf_folder, index_file
|
||||
)
|
||||
else:
|
||||
hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files)
|
||||
|
||||
|
||||
@@ -0,0 +1,437 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Clef-Omni (https://huggingface.co/Cloudflare/clef-omni).
|
||||
|
||||
The Qwen3-Omni thinker with a joint schema head that scores every option of
|
||||
every question in one forward pass. The head reads the final hidden states of
|
||||
the whole prompt, so it runs as a ``token_classify`` pooler.
|
||||
|
||||
Each request carries its schema in ``PoolingParams.extra_kwargs`` under
|
||||
``"joint_schema"``:
|
||||
|
||||
- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question.
|
||||
- ``question_spans``: the ``[start, end)`` of each question's instructions.
|
||||
- ``option_spans``: the ``[start, end)`` of each option, question by question.
|
||||
- ``option_counts``: the number of options of each question.
|
||||
|
||||
Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``),
|
||||
so they hold after multimodal placeholders are expanded. The output has one
|
||||
value per option, in ``option_spans`` order: its probability within its
|
||||
question, or its logit with ``use_activation=False``.
|
||||
"""
|
||||
|
||||
import math
|
||||
from collections.abc import Callable, Mapping, Set
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
from torch import nn
|
||||
from transformers.feature_extraction_utils import BatchFeature
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate
|
||||
from vllm.model_executor.layers.pooler.tokwise.methods import AllPool
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.tasks import PoolingTask
|
||||
from vllm.utils.torch_utils import async_tensor_h2d
|
||||
from vllm.v1.outputs import PoolerOutput
|
||||
from vllm.v1.pool.metadata import PoolingMetadata
|
||||
|
||||
from .interfaces_base import default_pooling_type
|
||||
from .qwen3_omni_moe_thinker import (
|
||||
Qwen3OmniMoeThinkerDummyInputsBuilder,
|
||||
Qwen3OmniMoeThinkerForConditionalGeneration,
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
Qwen3OmniMoeThinkerProcessingInfo,
|
||||
)
|
||||
|
||||
JOINT_HEAD_WEIGHTS = "joint_head.safetensors"
|
||||
NUM_QUESTION_TYPES = 3
|
||||
MAX_LOGIT_SCALE = math.log(100.0)
|
||||
|
||||
|
||||
def segment_mean(
|
||||
values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor
|
||||
) -> torch.Tensor:
|
||||
sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32)
|
||||
sums.index_add_(0, segment_ids, values.float())
|
||||
return (sums / lengths[:, None]).to(values.dtype)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SchemaIndices:
|
||||
"""Where one prompt's questions and options are, on the model's device."""
|
||||
|
||||
positions: torch.Tensor
|
||||
"""Every token of every span, questions first."""
|
||||
span_ids: torch.Tensor
|
||||
"""The span of each of ``positions``."""
|
||||
span_lengths: torch.Tensor
|
||||
"""``[Q + O]``."""
|
||||
question_types: torch.Tensor
|
||||
"""``[Q]``."""
|
||||
option_question: torch.Tensor
|
||||
"""The question of each option, ``[O]``."""
|
||||
option_slots: torch.Tensor
|
||||
"""Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``."""
|
||||
option_mask: torch.Tensor
|
||||
"""``[Q, max_options]``, true at the slots of options."""
|
||||
|
||||
|
||||
class EvidenceRoutingLayer(nn.Module):
|
||||
def __init__(self, width: int, heads: int, feedforward: int) -> None:
|
||||
super().__init__()
|
||||
self.query_norm = nn.LayerNorm(width)
|
||||
self.memory_norm = nn.LayerNorm(width)
|
||||
self.attention = nn.MultiheadAttention(width, heads, batch_first=True)
|
||||
self.feedforward_norm = nn.LayerNorm(width)
|
||||
# The Identity takes the place of the checkpoint's dropout.
|
||||
self.feedforward = nn.Sequential(
|
||||
nn.Linear(width, feedforward),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(feedforward, width),
|
||||
)
|
||||
|
||||
def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor:
|
||||
# Separate key and value tensors, like the reference, keep attention's
|
||||
# unpacked input projections.
|
||||
routed, _ = self.attention(
|
||||
self.query_norm(queries),
|
||||
self.memory_norm(memory),
|
||||
self.memory_norm(memory),
|
||||
need_weights=False,
|
||||
)
|
||||
queries = queries + routed
|
||||
return queries + self.feedforward(self.feedforward_norm(queries))
|
||||
|
||||
|
||||
class JointSchemaHead(nn.Module):
|
||||
def __init__(
|
||||
self,
|
||||
hidden_size: int,
|
||||
width: int,
|
||||
routing_layers: int,
|
||||
layers: int,
|
||||
heads: int,
|
||||
feedforward: int,
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.hidden_norm = nn.LayerNorm(hidden_size)
|
||||
self.memory_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_question_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.global_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_context_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False)
|
||||
self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width)
|
||||
self.evidence_layers = nn.ModuleList(
|
||||
EvidenceRoutingLayer(width, heads, feedforward)
|
||||
for _ in range(routing_layers)
|
||||
)
|
||||
self.option_summary_norm = nn.LayerNorm(width)
|
||||
self.layers = nn.ModuleList(
|
||||
nn.TransformerDecoderLayer(
|
||||
d_model=width,
|
||||
nhead=heads,
|
||||
dim_feedforward=feedforward,
|
||||
dropout=0.0,
|
||||
activation="gelu",
|
||||
batch_first=True,
|
||||
norm_first=True,
|
||||
)
|
||||
for _ in range(layers)
|
||||
)
|
||||
self.field_norm = nn.LayerNorm(width)
|
||||
self.option_norm = nn.LayerNorm(width)
|
||||
self.residual_scorer = nn.Sequential(
|
||||
nn.Linear(width * 4, width),
|
||||
nn.GELU(),
|
||||
nn.Identity(),
|
||||
nn.Linear(width, 1),
|
||||
)
|
||||
self.prior_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.joint_logit_scale = nn.Parameter(torch.zeros(()))
|
||||
self.residual_gate = nn.Parameter(torch.zeros(()))
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
lexical: torch.Tensor,
|
||||
schema: SchemaIndices,
|
||||
) -> torch.Tensor:
|
||||
"""The ``[O]`` option logits of one prompt, from its ``[L, hidden]``
|
||||
final hidden states and each option's ``[O, hidden]`` mean output
|
||||
embedding."""
|
||||
num_questions = schema.question_types.shape[0]
|
||||
option_question = schema.option_question
|
||||
hidden_states = self.hidden_norm(hidden_states)
|
||||
memory = self.memory_projection(hidden_states).unsqueeze(0)
|
||||
global_vector = hidden_states[-1]
|
||||
means = segment_mean(
|
||||
hidden_states[schema.positions], schema.span_ids, schema.span_lengths
|
||||
)
|
||||
question_vectors, contexts = means[:num_questions], means[num_questions:]
|
||||
|
||||
options = (
|
||||
self.option_context_projection(contexts)
|
||||
+ self.option_lexical_projection(lexical)
|
||||
+ self.option_question_projection(question_vectors)[option_question]
|
||||
).unsqueeze(0)
|
||||
for layer in self.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
options = options[0]
|
||||
|
||||
# Each field summarizes its options, weighted by their match to it.
|
||||
base_fields = self.question_projection(question_vectors)
|
||||
width = options.shape[-1]
|
||||
grid = options.new_zeros(schema.option_mask.numel(), width)
|
||||
grid[schema.option_slots] = options
|
||||
grid = grid.view(*schema.option_mask.shape, width)
|
||||
routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width)
|
||||
routing = routing.masked_fill(~schema.option_mask, float("-inf"))
|
||||
summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid)
|
||||
fields = (
|
||||
base_fields
|
||||
+ self.option_summary_norm(summaries)
|
||||
+ self.global_projection(global_vector).unsqueeze(0)
|
||||
+ self.type_embedding(schema.question_types)
|
||||
).unsqueeze(0)
|
||||
for layer in self.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = self.field_norm(fields[0])[option_question]
|
||||
|
||||
anchors = F.normalize(question_vectors + global_vector, dim=-1)
|
||||
prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
prior = prior_scale * (
|
||||
F.normalize(lexical, dim=-1) * anchors[option_question]
|
||||
).sum(dim=-1)
|
||||
options = self.option_norm(options)
|
||||
cosine = F.cosine_similarity(fields, options, dim=-1)
|
||||
features = torch.cat(
|
||||
[fields, options, fields * options, torch.abs(fields - options)], dim=-1
|
||||
)
|
||||
residual = self.residual_scorer(features).squeeze(-1)
|
||||
joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp()
|
||||
joint = joint_scale * cosine + residual
|
||||
return prior + torch.sigmoid(self.residual_gate) * joint
|
||||
|
||||
|
||||
def parse_joint_schema(
|
||||
extra_kwargs: dict[str, Any] | None, prompt_len: int
|
||||
) -> tuple[list[int], list[tuple[int, int]], list[int]] | None:
|
||||
"""The question types, the absolute spans of the questions then of the
|
||||
options, and the option counts. None if the schema is missing or invalid."""
|
||||
schema = (extra_kwargs or {}).get("joint_schema")
|
||||
if not isinstance(schema, dict):
|
||||
return None
|
||||
try:
|
||||
types = [int(t) for t in schema["question_types"]]
|
||||
counts = [int(c) for c in schema["option_counts"]]
|
||||
question_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["question_spans"]
|
||||
]
|
||||
option_spans = [
|
||||
(int(start) + prompt_len, int(end) + prompt_len)
|
||||
for start, end in schema["option_spans"]
|
||||
]
|
||||
except (KeyError, TypeError, ValueError):
|
||||
return None
|
||||
spans = question_spans + option_spans
|
||||
if (
|
||||
not types
|
||||
or len(counts) != len(types)
|
||||
or len(question_spans) != len(types)
|
||||
or len(option_spans) != sum(counts)
|
||||
or any(not 0 <= t < NUM_QUESTION_TYPES for t in types)
|
||||
or any(c < 1 for c in counts)
|
||||
or any(not 0 <= start < end <= prompt_len for start, end in spans)
|
||||
):
|
||||
return None
|
||||
return types, spans, counts
|
||||
|
||||
|
||||
class JointSchemaPooler(Pooler):
|
||||
"""Runs the joint schema head on each finished prompt.
|
||||
|
||||
It never raises, because it runs in the engine core: a prompt without a
|
||||
valid schema, such as a profiling dummy, gets an empty output.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
head: JointSchemaHead,
|
||||
embed_lexical: Callable[[torch.Tensor], torch.Tensor],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.pooling = AllPool()
|
||||
self.head = head
|
||||
self.embed_lexical = embed_lexical
|
||||
|
||||
def get_supported_tasks(self) -> Set[PoolingTask]:
|
||||
return {"token_classify"}
|
||||
|
||||
def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate:
|
||||
return PoolingParamsUpdate(requires_token_ids=True)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
pooling_metadata: PoolingMetadata,
|
||||
) -> PoolerOutput:
|
||||
pooled = self.pooling(hidden_states, pooling_metadata)
|
||||
prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu()
|
||||
outputs: list[torch.Tensor | None] = []
|
||||
for hidden, token_ids, params in zip(
|
||||
pooled, prompt_token_ids, pooling_metadata.pooling_params
|
||||
):
|
||||
if hidden is None:
|
||||
outputs.append(None)
|
||||
continue
|
||||
parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0])
|
||||
if parsed is None:
|
||||
outputs.append(hidden.new_empty(0, dtype=torch.float32))
|
||||
continue
|
||||
schema, lexical_ids, lexical_span_ids = self._indices(
|
||||
*parsed, hidden.device
|
||||
)
|
||||
lexical = segment_mean(
|
||||
self.embed_lexical(
|
||||
async_tensor_h2d(token_ids[lexical_ids], hidden.device)
|
||||
),
|
||||
lexical_span_ids,
|
||||
schema.span_lengths[len(parsed[0]) :],
|
||||
)
|
||||
logits = self.head(hidden, lexical, schema).float()
|
||||
if params.use_activation is not False:
|
||||
mask = schema.option_mask
|
||||
grid = logits.new_full((mask.numel(),), float("-inf"))
|
||||
grid[schema.option_slots] = logits
|
||||
probs = grid.view(mask.shape).softmax(dim=-1)
|
||||
logits = probs.view(-1)[schema.option_slots]
|
||||
outputs.append(logits)
|
||||
return outputs
|
||||
|
||||
@staticmethod
|
||||
def _indices(
|
||||
types: list[int],
|
||||
spans: list[tuple[int, int]],
|
||||
counts: list[int],
|
||||
device: torch.device,
|
||||
) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]:
|
||||
"""The schema's indices, plus the option tokens' positions (on the CPU)
|
||||
and their option spans, for the lexical embeddings."""
|
||||
num_questions = len(types)
|
||||
lengths = torch.tensor([end - start for start, end in spans])
|
||||
span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths)
|
||||
positions = torch.cat([torch.arange(start, end) for start, end in spans])
|
||||
num_question_tokens = int(lengths[:num_questions].sum())
|
||||
|
||||
counts_cpu = torch.tensor(counts)
|
||||
max_options = max(counts)
|
||||
option_question = torch.repeat_interleave(
|
||||
torch.arange(num_questions), counts_cpu
|
||||
)
|
||||
option_slots = torch.cat(
|
||||
[torch.arange(count) + q * max_options for q, count in enumerate(counts)]
|
||||
)
|
||||
schema = SchemaIndices(
|
||||
positions=async_tensor_h2d(positions, device),
|
||||
span_ids=async_tensor_h2d(span_ids, device),
|
||||
span_lengths=async_tensor_h2d(lengths, device),
|
||||
question_types=async_tensor_h2d(types, device, dtype=torch.long),
|
||||
option_question=async_tensor_h2d(option_question, device),
|
||||
option_slots=async_tensor_h2d(option_slots, device),
|
||||
option_mask=async_tensor_h2d(
|
||||
torch.arange(max_options) < counts_cpu[:, None], device
|
||||
),
|
||||
)
|
||||
lexical_span_ids = async_tensor_h2d(
|
||||
span_ids[num_question_tokens:] - num_questions, device
|
||||
)
|
||||
return schema, positions[num_question_tokens:], lexical_span_ids
|
||||
|
||||
|
||||
MAX_FRAME_PIXELS = 256 * 32 * 32
|
||||
|
||||
|
||||
def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor:
|
||||
"""The reference reads each video frame at at most 256 * 32 * 32 pixels,
|
||||
in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on
|
||||
their device (a CUDA tensor from torchcodec must not round-trip)."""
|
||||
height, width = frames.shape[1:3]
|
||||
if height * width <= MAX_FRAME_PIXELS:
|
||||
return frames
|
||||
scale = math.sqrt(MAX_FRAME_PIXELS / (width * height))
|
||||
new_width = max(2, round(width * scale / 2) * 2)
|
||||
new_height = max(2, round(height * scale / 2) * 2)
|
||||
is_tensor = isinstance(frames, torch.Tensor)
|
||||
tensor = frames if is_tensor else torch.from_numpy(frames)
|
||||
resized = F.interpolate(
|
||||
tensor.permute(0, 3, 1, 2).float(),
|
||||
size=(new_height, new_width),
|
||||
mode="bilinear",
|
||||
)
|
||||
if tensor.dtype == torch.uint8:
|
||||
resized = resized.round()
|
||||
resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous()
|
||||
return resized if is_tensor else resized.numpy()
|
||||
|
||||
|
||||
def _cap_video(item: Any) -> Any:
|
||||
if isinstance(item, tuple):
|
||||
frames, metadata = item
|
||||
return cap_frame_pixels(frames), metadata
|
||||
return cap_frame_pixels(item)
|
||||
|
||||
|
||||
class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor):
|
||||
"""Caps video frames at the reference's per-frame pixel count."""
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
hf_data: Mapping[str, object],
|
||||
hf_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
if videos := hf_data.get("videos"):
|
||||
assert isinstance(videos, list)
|
||||
hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]}
|
||||
return super()._call_hf_processor(hf_data, hf_kwargs)
|
||||
|
||||
|
||||
@default_pooling_type(tok_pooling_type="ALL")
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
ClefOmniMultiModalProcessor,
|
||||
info=Qwen3OmniMoeThinkerProcessingInfo,
|
||||
dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder,
|
||||
)
|
||||
class ClefOmniForJointSchema(Qwen3OmniMoeThinkerForConditionalGeneration):
|
||||
is_pooling_model = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config, prefix=prefix)
|
||||
model_config = vllm_config.model_config
|
||||
head_config = model_config.hf_config.joint_head_config
|
||||
self.pooler = JointSchemaPooler(
|
||||
JointSchemaHead(**head_config), self._embed_lexical
|
||||
)
|
||||
# The head's weights ship as their own safetensors file.
|
||||
self.secondary_weights = [
|
||||
DefaultModelLoader.Source(
|
||||
model_or_path=model_config.model,
|
||||
revision=model_config.revision,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=[JOINT_HEAD_WEIGHTS],
|
||||
)
|
||||
]
|
||||
|
||||
def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor:
|
||||
# The head embeds options with the output embeddings. ParallelLMHead
|
||||
# forbids lookups, so use the vocab-parallel embedding's.
|
||||
return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids)
|
||||
@@ -83,6 +83,7 @@ from vllm.multimodal.processing.processor import (
|
||||
PromptUpdate,
|
||||
PromptUpdateDetails,
|
||||
)
|
||||
from vllm.multimodal.utils import copy_mm_embedding_modality
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.transformers_utils.processor import cached_processor_from_config
|
||||
from vllm.utils.gpu_sync_debug import gpu_sync_allowed
|
||||
@@ -1498,8 +1499,12 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
|
||||
audio_in_video_item_idx += 1
|
||||
|
||||
# Match the fps the processor used, as HF and the mrope positions do.
|
||||
processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts")
|
||||
second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None)
|
||||
if second_per_grid_ts:
|
||||
if isinstance(processed_second_per_grid_ts, torch.Tensor):
|
||||
video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item()
|
||||
elif second_per_grid_ts:
|
||||
video_second_per_grid_t = second_per_grid_ts[item_idx]
|
||||
else:
|
||||
video_second_per_grid_t = 2.0
|
||||
@@ -1904,7 +1909,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
embeddings_main, embeddings_multiscale = torch.split(
|
||||
embeddings, [visual_dim, multi_dim], dim=-1
|
||||
)
|
||||
multimodal_embeddings[index] = embeddings_main
|
||||
multimodal_embeddings[index] = copy_mm_embedding_modality(
|
||||
embeddings, embeddings_main
|
||||
)
|
||||
multimodal_embeddings_multiscale.append(embeddings_multiscale)
|
||||
if not is_interleaved:
|
||||
current_positions = mm_positions[
|
||||
|
||||
@@ -287,6 +287,7 @@ _REWARD_MODELS = {
|
||||
|
||||
_TOKEN_CLASSIFICATION_MODELS = {
|
||||
"BertForTokenClassification": ("bert", "BertForTokenClassification"),
|
||||
"ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"),
|
||||
"ModernBertForTokenClassification": (
|
||||
"modernbert",
|
||||
"ModernBertForTokenClassification",
|
||||
|
||||
@@ -1570,3 +1570,67 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader):
|
||||
valid_frame_indices=valid_frame_indices,
|
||||
)
|
||||
return frames, metadata
|
||||
|
||||
|
||||
@VIDEO_LOADER_REGISTRY.register("clef_omni")
|
||||
class ClefOmniVideoBackend(VideoBackend):
|
||||
"""Clef-Omni's reference sampling: the first frame at or after each
|
||||
``1/fps`` boundary (frame i sits at ``i / original_fps`` seconds, so the
|
||||
k-th boundary is crossed by frame ``ceil(k * original_fps / fps)``),
|
||||
capped at 768 frames and padded to an even count by repeating the last
|
||||
frame."""
|
||||
|
||||
MAX_FRAMES = 768
|
||||
|
||||
@classmethod
|
||||
def compute_frames_index_to_sample(
|
||||
cls,
|
||||
source: VideoSourceMetadata,
|
||||
target: VideoTargetMetadata,
|
||||
**kwargs,
|
||||
) -> list[int]:
|
||||
if source.original_fps <= 0:
|
||||
raise ValueError("video has an unknown frame rate")
|
||||
step = source.original_fps / target.fps
|
||||
indices = []
|
||||
k = 0
|
||||
while (index := math.ceil(k * step - 1e-6)) < source.total_frames_num:
|
||||
indices.append(index)
|
||||
if len(indices) > cls.MAX_FRAMES:
|
||||
raise ValueError(
|
||||
f"video is longer than {cls.MAX_FRAMES / target.fps:.0f} seconds"
|
||||
)
|
||||
k += 1
|
||||
if not indices:
|
||||
raise ValueError("video has no decodable frames")
|
||||
return indices
|
||||
|
||||
@classmethod
|
||||
def load_bytes(
|
||||
cls,
|
||||
data: bytes,
|
||||
num_frames: int = -1,
|
||||
fps: int = 2,
|
||||
max_duration: int = 300,
|
||||
frame_recovery: bool = False,
|
||||
*,
|
||||
backend: VideoDecoderBackend = "opencv",
|
||||
**kwargs: Any,
|
||||
) -> tuple[DecodedFrames, dict[str, Any]]:
|
||||
frames, metadata = super().load_bytes(
|
||||
data,
|
||||
num_frames=num_frames,
|
||||
fps=fps,
|
||||
max_duration=max_duration,
|
||||
frame_recovery=frame_recovery,
|
||||
backend=backend,
|
||||
**kwargs,
|
||||
)
|
||||
if len(frames) % 2:
|
||||
last = frames[-1:]
|
||||
if isinstance(frames, torch.Tensor):
|
||||
frames = torch.cat([frames, last])
|
||||
else:
|
||||
frames = np.concatenate([frames, last])
|
||||
metadata["frames_indices"].append(metadata["frames_indices"][-1])
|
||||
return frames, metadata
|
||||
|
||||
@@ -17,6 +17,10 @@ from vllm.transformers_utils.config import (
|
||||
ConfigFormat,
|
||||
get_safetensors_params_metadata,
|
||||
)
|
||||
from vllm.transformers_utils.repo_utils import (
|
||||
file_or_path_exists,
|
||||
get_hf_file_to_dict,
|
||||
)
|
||||
from vllm.utils.torch_utils import common_broadcastable_dtype
|
||||
|
||||
logger = init_logger(__name__)
|
||||
@@ -643,6 +647,31 @@ class MimoV2ModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
_strip_mimo_v2_attention_chunk_size(hf_config, hf_text_config)
|
||||
|
||||
|
||||
class Qwen3OmniMoeModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def __init__(
|
||||
self,
|
||||
hf_config: PreTrainedConfig,
|
||||
hf_text_config: PreTrainedConfig,
|
||||
revision: str | None = None,
|
||||
):
|
||||
# Clef-Omni keeps Qwen3-Omni's config and adds a joint schema head;
|
||||
# route to its wrapper class when the head's config file is present.
|
||||
# Mutate hf_config (like the deepseek_v4/mimo_v2 rewrites above)
|
||||
# because model-class resolution reads hf_config.architectures.
|
||||
if getattr(hf_config, "architectures", None) == [
|
||||
"Qwen3OmniMoeForConditionalGeneration"
|
||||
]:
|
||||
model_path = hf_config.name_or_path
|
||||
if model_path and file_or_path_exists(
|
||||
model_path, "joint_head_config.json", revision
|
||||
):
|
||||
hf_config.architectures = ["ClefOmniForJointSchema"]
|
||||
hf_config.joint_head_config = get_hf_file_to_dict(
|
||||
"joint_head_config.json", model_path, revision
|
||||
)
|
||||
super().__init__(hf_config, hf_text_config, revision)
|
||||
|
||||
|
||||
class MimoV2MTPModelArchConfigConvertor(ModelArchConfigConvertorBase):
|
||||
def __init__(
|
||||
self,
|
||||
@@ -822,6 +851,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = {
|
||||
"bailing_hybrid_v3_mtp": BailingHybridV3MTPModelArchConfigConvertor,
|
||||
"pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor,
|
||||
"qwen3_5_mtp": Qwen3_5MTPModelArchConfigConvertor,
|
||||
"qwen3_omni_moe": Qwen3OmniMoeModelArchConfigConvertor,
|
||||
"qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor,
|
||||
"qwen4_exp_mtp": Qwen4ExpMTPModelArchConfigConvertor,
|
||||
"RefinedWeb": FalconModelArchConfigConvertor,
|
||||
|
||||
Reference in New Issue
Block a user