From 092c35dd155db2644da6bb671332c51902bd3039 Mon Sep 17 00:00:00 2001 From: Jiangyun Zhu Date: Mon, 12 Oct 2026 21:27:34 +0800 Subject: [PATCH] [Model] Support Cloudflare/clef-omni on /v1/systemone (#61064) Signed-off-by: zjy0516 Signed-off-by: Isotr0py Co-authored-by: Claude Opus 5.5 Co-authored-by: Isotr0py --- .../online_serving/structured_decisions.md | 27 ++ .../structured_decisions/test_strategies.py | 345 +++++++++++++- .../test_filter_duplicate_safetensors.py | 29 ++ .../multimodal/pooling/test_clef_omni.py | 168 +++++++ .../multimodal/processing/test_qwen3_omni.py | 30 ++ tests/models/registry.py | 4 + tests/multimodal/test_video.py | 16 + vllm/entrypoints/generate/api_router.py | 39 -- .../structured_decisions/api_router.py | 39 ++ .../generate/structured_decisions/protocol.py | 9 + .../generate/structured_decisions/serving.py | 23 +- .../structured_decisions/strategies.py | 362 ++++++++++++++- .../launchers/api_server/app_state.py | 13 + .../launchers/api_server/routers.py | 6 + .../model_loader/default_loader.py | 8 +- vllm/model_executor/models/clef_omni.py | 437 ++++++++++++++++++ .../models/qwen3_omni_moe_thinker.py | 11 +- vllm/model_executor/models/registry.py | 1 + vllm/multimodal/video.py | 64 +++ .../model_arch_config_convertor.py | 30 ++ 20 files changed, 1592 insertions(+), 69 deletions(-) create mode 100644 tests/models/multimodal/pooling/test_clef_omni.py create mode 100644 vllm/model_executor/models/clef_omni.py diff --git a/docs/serving/online_serving/structured_decisions.md b/docs/serving/online_serving/structured_decisions.md index a9ed5a2ee5..4d8d65d89e 100644 --- a/docs/serving/online_serving/structured_decisions.md +++ b/docs/serving/online_serving/structured_decisions.md @@ -130,6 +130,32 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a | options per `choice` | 26 | | levels per `score` | 10 | +## Clef-Omni + +[Clef-Omni](https://huggingface.co/Cloudflare/clef-omni) is a decision model: +a joint schema head on the Qwen3-Omni thinker scores every option of every +question in one forward pass. vLLM serves it as a pooling model. + +```bash +vllm serve Cloudflare/clef-omni +``` + +- A request is one read. The prompt is the one the checkpoint's + `joint_schema_model.py` builds: the state, then every question with its + options. A JSON state is written as compact JSON with sorted keys. +- A state too long for the model is truncated to fit, as the reference does. + A request whose questions alone exceed 64,000 tokens, or `max_model_len`, + returns 400. +- `instructions` returns 400, since the prompt is fixed. `chat_template_kwargs` + has no effect. +- There are no labels, so `label_mass` is 1 and `confidence` is the chosen + option's probability. `usage.input_tokens` counts the prompt once, and + `output_tokens` is 0. +- `images`, `audio` and `videos` are read as the reference does: placed + before the state, videos sampled at 2 fps, and a video's soundtrack heard + with it when every video has one. Audio clips cannot be sent together with + videos that have soundtracks. + ## Request fields | field | meaning | @@ -139,6 +165,7 @@ the request lists them, a `noul` labels its options `yes` and `no`, and a | `instructions` | optional context placed ahead of the questions | | `chat_template_kwargs` | passed to the chat template, for example `{"enable_thinking": false}` | | `cache_salt` | optional [prefix-cache salt](../../usage/security.md#cache-salting) applied to every question | +| `images`, `audio`, `videos` | optional media URLs or data URLs, for models that take them ([Clef-Omni](#clef-omni)) | A question with any field other than `type`, `instructions` and `criteria` is rejected with a 400. diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 41b9a8f012..8d38e61e62 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -1,12 +1,18 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import json +from io import BytesIO from types import SimpleNamespace from typing import Any from unittest.mock import MagicMock +import numpy as np +import pybase64 as base64 import pytest +import torch from fastapi import FastAPI, Request from fastapi.testclient import TestClient +from PIL import Image from vllm.config import ModelConfig from vllm.engine.protocol import EngineClient @@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import ( ) from vllm.entrypoints.generate.structured_decisions.serving import ( ServingStructuredDecisions, - state_text, + parse_questions, ) from vllm.entrypoints.generate.structured_decisions.strategies import ( DiffusionGemmaCanvasStrategy, + JointSchemaStrategy, NextTokenStrategy, ReadContext, reply_label_ids, select_read_strategy, + state_text, ) from vllm.entrypoints.openai.models.protocol import BaseModelPath from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.exception_handling.register import ( init_exception_handler, ) +from vllm.inputs import mm_input from vllm.logprobs import Logprob -from vllm.outputs import CompletionOutput, RequestOutput +from vllm.multimodal.media import MediaConnector, MediaWithBytes +from vllm.outputs import ( + CompletionOutput, + PoolingOutput, + PoolingRequestOutput, + RequestOutput, +) from vllm.renderers.hf import HfRenderer from vllm.renderers.online_renderer import OnlineRenderer @@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any: def test_strategy_selection(): - qwen = "Qwen3ForCausalLM" - assert select_read_strategy(model(qwen)) is NextTokenStrategy - assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy + qwen, generate = "Qwen3ForCausalLM", ("generate",) + assert select_read_strategy(model(qwen), generate) is NextTokenStrategy assert ( - select_read_strategy(model("DiffusionGemmaForBlockDiffusion")) + select_read_strategy(model(qwen, "processed_logprobs"), generate) + is NextTokenStrategy + ) + assert ( + select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate) is DiffusionGemmaCanvasStrategy ) with pytest.raises(ValueError, match="does not support LlamaForCausalLM"): - select_read_strategy(model("LlamaForCausalLM")) + select_read_strategy(model("LlamaForCausalLM"), generate) with pytest.raises(ValueError, match="not raw_logits"): - select_read_strategy(model(qwen, "raw_logits")) + select_read_strategy(model(qwen, "raw_logits"), generate) + # A generative architecture served as a pooling model has no read strategy. + with pytest.raises(ValueError, match="need the generate task"): + select_read_strategy(model(qwen), ("embed",)) + # Clef-Omni scores options with its head, so it reads no logprobs. + clef = model("ClefOmniForJointSchema", "raw_logits") + assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy def test_route_is_registered_by_default(): @@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen): @pytest.fixture -def serving_and_engine(qwen): +def engine_and_renderer(qwen): tokenizer, _ = qwen model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32") renderer = HfRenderer( @@ -148,6 +173,19 @@ def serving_and_engine(qwen): chat_template=None, chat_template_content_format="auto", ) + yield engine, online_renderer + renderer.shutdown() + + +def serving(engine, strategy) -> ServingStructuredDecisions: + name = engine.model_config.model + models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)]) + return ServingStructuredDecisions(models, strategy) + + +@pytest.fixture +def serving_and_engine(engine_and_renderer): + engine, online_renderer = engine_and_renderer async def generate(prompt, params, request_id, **kwargs): label_ids = params.logprob_token_ids @@ -171,11 +209,7 @@ def serving_and_engine(qwen): engine.generate.side_effect = generate strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {})) - models = OpenAIServingModels( - engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)] - ) - yield ServingStructuredDecisions(models, strategy), engine - renderer.shutdown() + return serving(engine, strategy), engine @pytest.mark.asyncio @@ -256,3 +290,286 @@ def test_canvas_read(): strategy._read_input( {"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4] ) + + +JOINT_SCHEMA_PROMPT = """<|im_start|>system +Read the complete state and schema. Decide every field jointly. Each answer \ +must be exactly one of that field's allowed options.<|im_end|> +<|im_start|>user +STATE: +{"customer":{"tier":"gold"},"ticket":"Charged twice"} + +SCHEMA FIELDS: + +FIELD 1 +ID: team +TYPE: choice +INSTRUCTION: Which team? +ALLOWED OPTIONS: +OPTION 1: {"description":"payments","option_id":"billing"} +OPTION 2: {"option_id":"security"} +OPTION 3: {"description":"deliveries","option_id":"shipping"} +END FIELD + +FIELD 2 +ID: urgent +TYPE: noul +INSTRUCTION: urgent +ALLOWED OPTIONS: +OPTION 1: {"description":"money is moving","option_id":"true"} +OPTION 2: {"description":"The proposition is false or the answer is no.",\ +"option_id":"false"} +END FIELD + +FIELD 3 +ID: tone +TYPE: score +INSTRUCTION: How upset? +ALLOWED OPTIONS: +OPTION 1: {"description":"calm","option_id":"0"} +OPTION 2: {"description":"annoyed","option_id":"1"} +OPTION 3: {"description":"furious","option_id":"2"} +END FIELD + +<|im_end|> +<|im_start|>assistant + + + + +JOINT SCHEMA DECISIONS:""" + +JOINT_SCHEMA_REQUEST: dict[str, Any] = dict( + state={"ticket": "Charged twice", "customer": {"tier": "gold"}}, + questions={ + "team": { + "type": "choice", + "instructions": "Which team?", + "criteria": { + "shipping": "deliveries", + "billing": "payments", + "security": None, + }, + }, + "urgent": {"type": "noul", "criteria": {"true": "money is moving"}}, + "tone": { + "type": "score", + "instructions": "How upset?", + "criteria": ["calm", "annoyed", "furious"], + }, + }, +) + + +@pytest.fixture +def joint_schema(qwen, engine_and_renderer): + tokenizer, _ = qwen + engine, online_renderer = engine_and_renderer + # The probability the head gives each option, by option id. + scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2} + scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3} + + async def encode(prompt, params, request_id, **kwargs): + ids = prompt["prompt_token_ids"] + spans = params.extra_kwargs["joint_schema"]["option_spans"] + options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans] + data = torch.tensor([scores[o["option_id"]] for o in options]) + yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True) + + engine.encode.side_effect = encode + strategy = JointSchemaStrategy( + ReadContext(engine, online_renderer, None, "auto", {}) + ) + return serving(engine, strategy), engine, tokenizer + + +@pytest.mark.asyncio +async def test_joint_schema_reads_every_question_at_once(joint_schema): + serving, engine, tokenizer = joint_schema + response = await serving.create_decision( + StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt") + ) + assert isinstance(response, StructuredDecisionResponse) + + engine.encode.assert_called_once() + engine.check_admission.assert_called_once_with(1) + prompt, params = engine.encode.call_args.args[:2] + ids = prompt["prompt_token_ids"] + assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT + assert prompt["cache_salt"] == "salt" + assert params.task == "token_classify" + schema = params.extra_kwargs["joint_schema"] + assert schema["question_types"] == [1, 0, 2] + assert schema["option_counts"] == [3, 2, 3] + assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [ + "Which team?", + "urgent", + "How upset?", + ] + + # Options are read in the model's order and answered in the request's. + team = response.answers["team"] + assert team["choice"] == "billing" + assert list(team["probabilities"]) == ["shipping", "billing", "security"] + assert team["probabilities"] == pytest.approx( + {"shipping": 0.2, "billing": 0.7, "security": 0.1} + ) + assert response.answers["urgent"]["noul"] == pytest.approx(0.9) + assert response.answers["tone"]["score"] == pytest.approx(1.1) + assert response.usage.input_tokens == len(ids) + assert response.usage.output_tokens == 0 + assert all(d.label_mass == 1.0 for d in response.diagnostics.values()) + + +@pytest.mark.asyncio +async def test_joint_schema_rejects_instructions(joint_schema): + serving, engine, _ = joint_schema + response = await serving.create_decision( + StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.") + ) + assert isinstance(response, ErrorResponse) + assert response.error.code == 400 + engine.encode.assert_not_called() + + +def test_joint_schema_truncates_the_state(joint_schema): + serving, _, _ = joint_schema + strategy = serving.strategy + request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST) + questions = parse_questions(request, strategy.limits()) + prompt, schema, _ = strategy.encode(questions, "word " * 100) + strategy.max_length = len(prompt) - 50 + truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100) + # The state loses 50 tokens and the questions stay at the end. + assert len(truncated) == strategy.max_length + assert truncated[-200:] == prompt[-200:] + assert truncated_schema == schema + strategy.max_length = 100 + with pytest.raises(StructuredDecisionError, match="questions take"): + strategy.encode(questions, "word") + + +@pytest.mark.asyncio +async def test_media_needs_a_model_that_takes_it(serving_and_engine): + serving, engine = serving_and_engine + request = StructuredDecisionRequest( + **JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"] + ) + response = await serving.create_decision(request) + assert isinstance(response, ErrorResponse) + assert response.error.code == 400 + assert "does not take image inputs" in response.error.message + engine.generate.assert_not_called() + + +@pytest.mark.asyncio +async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch): + serving, engine, tokenizer = joint_schema + strategy = serving.strategy + strategy.__dict__["_connector"] = MediaConnector() + image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>") + rendered: dict[str, Any] = {} + + async def process_for_engine_async(prompt, arrival_time): + rendered.update(prompt) + # The image's placeholder expands to three tokens. + ids = [ + token + for pad in prompt["prompt_token_ids"] + for token in ([pad] * 3 if pad == image_pad else [pad]) + ] + return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt")) + + renderer = strategy.context.online_renderer.renderer + monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async) + buffer = BytesIO() + Image.new("RGB", (8, 8), "red").save(buffer, format="PNG") + image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode() + response = await serving.create_decision( + StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image]) + ) + assert isinstance(response, StructuredDecisionResponse) + + placeholder = tokenizer.encode( + "<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False + ) + assert rendered["prompt_token_ids"] == strategy.prefix + placeholder + assert rendered["multi_modal_data"]["image"][0].size == (8, 8) + assert rendered["mm_processor_kwargs"] == { + "images_kwargs": {"size": strategy.IMAGE_SIZE} + } + prompt = engine.encode.call_args.args[0] + assert prompt["type"] == "multimodal" + ids = prompt["prompt_token_ids"] + expanded = len(strategy.prefix) + len(placeholder) + 2 + assert ids[:expanded].count(image_pad) == 3 + state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1] + assert tokenizer.decode(ids[expanded:]) == state_and_questions + assert response.answers["team"]["choice"] == "billing" + assert response.usage.input_tokens == len(ids) + + +@pytest.mark.asyncio +async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch): + serving, engine, tokenizer = joint_schema + strategy = serving.strategy + video = MediaWithBytes( + (np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes" + ) + track = (np.zeros(160, np.float32), 16000) + rendered: dict[str, Any] = {} + + class FakeConnector: + async def fetch_image_async(self, url): + raise AssertionError("no images in this test") + + async def load_from_url_async(self, url, media_io, *, fetch_timeout=None): + return video + + async def fetch_audio_async(self, url): + if url == "https://example.org/silent.mp4": + raise ValueError("No audio stream found.") + return track + + strategy.__dict__["_connector"] = FakeConnector() + + async def process_for_engine_async(prompt, arrival_time): + rendered.clear() + rendered.update(prompt) + return mm_input(prompt["prompt_token_ids"], {}, {}, {}) + + renderer = strategy.context.online_renderer.renderer + monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async) + + response = await serving.create_decision( + StructuredDecisionRequest( + **JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"] + ) + ) + assert isinstance(response, StructuredDecisionResponse) + assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]} + assert rendered["mm_processor_kwargs"] == { + "images_kwargs": {"size": strategy.IMAGE_SIZE}, + "use_audio_in_video": True, + } + + # A video without a decodable soundtrack is watched silently. + response = await serving.create_decision( + StructuredDecisionRequest( + **JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"] + ) + ) + assert isinstance(response, StructuredDecisionResponse) + assert rendered["multi_modal_data"] == {"video": [video]} + assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False + + # Audio clips cannot be read with videos that have soundtracks. + response = await serving.create_decision( + StructuredDecisionRequest( + **JOINT_SCHEMA_REQUEST, + videos=["https://example.org/talking.mp4"], + audio=["https://example.org/clip.wav"], + ) + ) + assert isinstance(response, ErrorResponse) + assert response.error.code == 400 diff --git a/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py b/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py index da04d0108f..7aca4b64ab 100644 --- a/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py +++ b/tests/model_executor/model_loader/test_filter_duplicate_safetensors.py @@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch ["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"], sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")), ] + + +def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path): + """A source with explicit allow patterns (e.g. a standalone head file in + the repo root) must not be validated against the backbone's index.""" + save_file( + {"backbone.weight": torch.zeros(1)}, + tmp_path / "model-00001-of-00001.safetensors", + ) + save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors") + (tmp_path / "model.safetensors.index.json").write_text( + json.dumps( + {"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}} + ) + ) + + loader = DefaultModelLoader(LoadConfig()) + secondary = DefaultModelLoader.Source( + model_or_path=str(tmp_path), + revision=None, + prefix="pooler.head.", + allow_patterns_overrides=["joint_head.safetensors"], + ) + assert [n for n, _ in loader._get_weights_iterator(secondary)] == [ + "pooler.head.head.weight" + ] + # The primary source still filters files the backbone index doesn't list. + primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None) + assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"] diff --git a/tests/models/multimodal/pooling/test_clef_omni.py b/tests/models/multimodal/pooling/test_clef_omni.py new file mode 100644 index 0000000000..ccdc913b31 --- /dev/null +++ b/tests/models/multimodal/pooling/test_clef_omni.py @@ -0,0 +1,168 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""The Clef-Omni pooler against a port of the checkpoint's joint schema head, +which loops over questions where the pooler batches them.""" + +import math +from types import SimpleNamespace +from unittest.mock import patch + +import pytest +import torch +import torch.nn.functional as F + +from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler +from vllm.pooling_params import PoolingParams +from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates + +HIDDEN, VOCAB = 64, 500 +TYPES = [0, 1, 1, 2] +COUNTS = [2, 3, 1, 5] + + +def reference_logits(head, hidden, token_ids, embedding, question_spans, spans): + """``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py + for one record, with ``spans`` the option spans of each question.""" + hidden = head.hidden_norm(hidden) + memory = head.memory_projection(hidden).unsqueeze(0) + global_vector = hidden[-1] + questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans]) + contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans] + lexicals = [ + torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans + ] + options = torch.cat( + [ + head.option_context_projection(c) + + head.option_lexical_projection(lex) + + head.option_question_projection(questions[i]).unsqueeze(0) + for i, (c, lex) in enumerate(zip(contexts, lexicals)) + ] + ).unsqueeze(0) + for layer in head.evidence_layers: + options = layer(options, memory) + routed = torch.split(options[0], [len(q) for q in spans]) + + base = head.question_projection(questions) + summaries = [] + for field, opts in zip(base, routed): + weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0) + summaries.append((weights[:, None] * opts).sum(0)) + fields = ( + base + + head.option_summary_norm(torch.stack(summaries)) + + head.global_projection(global_vector) + + head.type_embedding(torch.tensor(TYPES)) + ).unsqueeze(0) + for layer in head.layers: + fields = layer(fields, memory) + fields = head.field_norm(fields[0]) + + logits = [] + for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)): + anchor = F.normalize(questions[i] + global_vector, dim=-1) + prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp() + prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor) + opts = head.option_norm(opts) + field = field.expand_as(opts) + features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1) + joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp() + joint = joint_scale * F.cosine_similarity(field, opts, dim=-1) + joint = joint + head.residual_scorer(features).squeeze(-1) + logits.append(prior + torch.sigmoid(head.residual_gate) * joint) + return logits + + +@pytest.fixture +def pooler(): + torch.manual_seed(0) + head = JointSchemaHead( + hidden_size=HIDDEN, + width=32, + routing_layers=2, + layers=2, + heads=4, + feedforward=48, + ).eval() + with torch.no_grad(): + for p in head.parameters(): + p.add_(torch.randn_like(p) * 0.2) + embedding = torch.randn(VOCAB, HIDDEN) + config = SimpleNamespace( + scheduler_config=SimpleNamespace( + enable_chunked_prefill=False, async_scheduling=False + ) + ) + with patch( + "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config", + return_value=config, + ): + pooler = JointSchemaPooler(head, lambda ids: embedding[ids]) + return pooler, embedding + + +def schema(length: int): + """Questions after a 20-token state, with options of 1 to 4 tokens.""" + pos = 20 + question_spans: list[tuple[int, int]] = [] + spans: list[list[tuple[int, int]]] = [] + for count in COUNTS: + question_spans.append((pos, pos + 4)) + pos += 6 + spans.append([]) + for k in range(count): + n = 1 + (k * 3) % 4 + spans[-1].append((pos, pos + n)) + pos += n + 1 + assert pos < length + extra = { + "question_types": TYPES, + "question_spans": [[s - length, e - length] for s, e in question_spans], + "option_spans": [[s - length, e - length] for q in spans for s, e in q], + "option_counts": COUNTS, + } + return {"joint_schema": extra}, question_spans, spans + + +@torch.inference_mode() +def test_pooler_matches_reference(pooler): + pooler, embedding = pooler + lengths = [120, 90, 100] + token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths] + hidden = torch.randn(sum(lengths), HIDDEN) + first, first_questions, first_spans = schema(lengths[0]) + last, last_questions, last_spans = schema(lengths[2]) + params = [ + PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first), + # A prompt without a schema, like the profiling run's. + PoolingParams(task="token_classify", use_activation=True), + PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last), + ] + padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long) + for i, ids in enumerate(token_ids): + padded[i, : len(ids)] = ids + metadata = PoolingMetadata( + prompt_lens=torch.tensor(lengths), + prompt_token_ids=None, + prompt_token_ids_cpu=padded, + pooling_params=params, + pooling_states=[PoolingStates() for _ in lengths], + ) + metadata.build_pooling_cursor( + torch.tensor(lengths).numpy(), + seq_lens_cpu=torch.tensor(lengths), + device=torch.device("cpu"), + ) + probs, empty, logits = pooler(hidden, metadata) + + first_hidden, _, last_hidden = torch.split(hidden, lengths) + expected = reference_logits( + pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans + ) + expected_probs = torch.cat([q.softmax(-1) for q in expected]) + torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4) + assert empty.numel() == 0 + expected = reference_logits( + pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans + ) + torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4) diff --git a/tests/models/multimodal/processing/test_qwen3_omni.py b/tests/models/multimodal/processing/test_qwen3_omni.py index 301471f45a..c0ebf02971 100644 --- a/tests/models/multimodal/processing/test_qwen3_omni.py +++ b/tests/models/multimodal/processing/test_qwen3_omni.py @@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None: f"Expected longer audio (2s) to have more tokens than shorter (1s). " f"Got short={short_tokens}, long={long_tokens}" ) + + +@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"]) +def test_audio_in_video_follows_requested_fps(model_id: str) -> None: + """Audio and video tokens interleave by the fps the video was processed + at, as in HF, not by the default of one frame a second.""" + ctx = build_model_context( + model_id, + limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1}, + ) + processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config) + rng = np.random.RandomState(42) + video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8) + audio = rng.rand(32000).astype(np.float32) + prompt = "<|vision_start|><|video_pad|><|vision_end|>" + mm_kwargs: dict[str, Any] = { + "use_audio_in_video": True, + "videos_kwargs": {"fps": 2.0, "do_sample_frames": False}, + } + processed = processor( + prompt, + mm_items=processor.info.parse_mm_data( + {"video": [video], "audio": [(audio, 16000)]} + ), + hf_processor_mm_kwargs=mm_kwargs, + ) + expected = processor.info.get_hf_processor()( + text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs + ) + assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist() diff --git a/tests/models/registry.py b/tests/models/registry.py index 2ed6d4871c..0a7027461e 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -1351,6 +1351,10 @@ _MULTIMODAL_EXAMPLE_MODELS = { "Qwen/Qwen3-Omni-30B-A3B-Instruct", max_model_len=4096, ), + "ClefOmniForJointSchema": _HfExamplesInfo( + "Cloudflare/clef-omni", + max_model_len=4096, + ), "Qwen3ASRForConditionalGeneration": _HfExamplesInfo( "Qwen/Qwen3-ASR-0.6B", max_model_len=4096, diff --git a/tests/multimodal/test_video.py b/tests/multimodal/test_video.py index 407bffed20..0066d239ea 100644 --- a/tests/multimodal/test_video.py +++ b/tests/multimodal/test_video.py @@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend from vllm.multimodal.video import ( PYNVVIDEOCODEC_VIDEO_BACKEND, VIDEO_LOADER_REGISTRY, + ClefOmniVideoBackend, DynamicVideoBackend, Glm5NextVideoBackend, GLM46VVideoBackend, @@ -1813,3 +1814,18 @@ def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path): assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1 # One initial seek, then pure walking -- no re-seek churn. assert cap.seeks == 1 + + +def test_clef_omni_backend_samples_reference_frames(tmp_path): + """Clef-Omni samples the first frame at or after each 0.5 s boundary + (the reference's 2 fps timestamp sampling), not uniform strides.""" + pytest.importorskip("cv2") + + # 2 s at 30 fps: the reference reads frames 0, 15, 30 and 45, where + # uniform sampling would read 0, 19, 39 and 59. + path = _write_gray_video(tmp_path, total_frames=60, fps=30) + frames, metadata = ClefOmniVideoBackend.load_bytes(path.read_bytes()) + assert metadata["frames_indices"] == [0, 15, 30, 45] + assert metadata["do_sample_frames"] is False + for frame, idx in zip(frames, [0, 15, 30, 45]): + assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1 diff --git a/vllm/entrypoints/generate/api_router.py b/vllm/entrypoints/generate/api_router.py index 6bf8d69026..9bbe4e879b 100644 --- a/vllm/entrypoints/generate/api_router.py +++ b/vllm/entrypoints/generate/api_router.py @@ -5,7 +5,6 @@ from typing import TYPE_CHECKING, Any from fastapi import FastAPI import vllm.envs as envs -from vllm.logger import init_logger if TYPE_CHECKING: from argparse import Namespace @@ -18,8 +17,6 @@ if TYPE_CHECKING: else: RequestLogger = object -logger = init_logger(__name__) - def register_generate_api_routers(app: FastAPI): from vllm.entrypoints.openai.chat_completion.api_router import ( @@ -56,12 +53,6 @@ def register_generate_api_routers(app: FastAPI): register_generative_scoring_api_router(app) - from .structured_decisions.api_router import ( - register_structured_decisions_api_router, - ) - - register_structured_decisions_api_router(app) - async def init_generate_state( engine_client: "EngineClient", @@ -231,33 +222,3 @@ async def init_generate_state( state.openai_serving_models, request_logger=request_logger, ) - - from .structured_decisions.serving import ServingStructuredDecisions - from .structured_decisions.strategies import ReadContext, select_read_strategy - - strategy = None - if "generate" in supported_tasks: - try: - strategy_cls = select_read_strategy(engine_client.model_config) - strategy = strategy_cls( - ReadContext( - engine_client=engine_client, - online_renderer=state.online_renderer, - chat_template=resolved_chat_template, - chat_template_content_format=args.chat_template_content_format, - default_chat_template_kwargs=default_chat_template_kwargs, - ) - ) - except ValueError as e: - # Info, since every model without a read strategy lands here. - logger.info("/v1/systemone is disabled: %s", e) - strategy = None - state.serving_structured_decisions = ( - ServingStructuredDecisions( - state.openai_serving_models, - strategy, - request_logger=request_logger, - ) - if strategy is not None - else None - ) diff --git a/vllm/entrypoints/generate/structured_decisions/api_router.py b/vllm/entrypoints/generate/structured_decisions/api_router.py index 5729b58b2e..a3afe7fe11 100644 --- a/vllm/entrypoints/generate/structured_decisions/api_router.py +++ b/vllm/entrypoints/generate/structured_decisions/api_router.py @@ -1,19 +1,30 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from argparse import Namespace from http import HTTPStatus +from typing import Any from fastapi import APIRouter, Depends, FastAPI, Request from fastapi.responses import JSONResponse +from starlette.datastructures import State +from vllm.engine.protocol import EngineClient from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, validate_json_request, with_cancellation, ) +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.logger import init_logger +from vllm.renderers.chat_utils import load_chat_template +from vllm.tasks import SupportedTask from .protocol import StructuredDecisionRequest from .serving import ServingStructuredDecisions +from .strategies import ReadContext, select_read_strategy + +logger = init_logger(__name__) router = APIRouter() @@ -50,3 +61,31 @@ async def create_structured_decision( def register_structured_decisions_api_router(app: FastAPI): app.include_router(router) + + +def init_structured_decisions_state( + engine_client: EngineClient, + state: State, + args: Namespace, + request_logger: RequestLogger | None, + supported_tasks: tuple[SupportedTask, ...], + default_chat_template_kwargs: dict[str, Any], +): + state.serving_structured_decisions = None + try: + strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks) + strategy = strategy_cls( + ReadContext( + engine_client=engine_client, + online_renderer=state.online_renderer, + chat_template=load_chat_template(args.chat_template), + chat_template_content_format=args.chat_template_content_format, + default_chat_template_kwargs=default_chat_template_kwargs, + ) + ) + except ValueError as e: + logger.debug("/v1/systemone is disabled: %s", e) + return + state.serving_structured_decisions = ServingStructuredDecisions( + state.openai_serving_models, strategy, request_logger=request_logger + ) diff --git a/vllm/entrypoints/generate/structured_decisions/protocol.py b/vllm/entrypoints/generate/structured_decisions/protocol.py index ab89084dff..a8f104b8cb 100644 --- a/vllm/entrypoints/generate/structured_decisions/protocol.py +++ b/vllm/entrypoints/generate/structured_decisions/protocol.py @@ -30,6 +30,15 @@ class StructuredDecisionRequest(OpenAIBaseModel): instructions: str | None = Field( default=None, description="Context placed ahead of the questions." ) + images: list[str] | None = Field( + default=None, description="Image URLs or data URLs, for models that see." + ) + audio: list[str] | None = Field( + default=None, description="Audio URLs or data URLs, for models that hear." + ) + videos: list[str] | None = Field( + default=None, description="Video URLs or data URLs, for models that see." + ) chat_template_kwargs: dict[str, Any] | None = None cache_salt: str | None = Field( default=None, diff --git a/vllm/entrypoints/generate/structured_decisions/serving.py b/vllm/entrypoints/generate/structured_decisions/serving.py index 2633647a0b..52fdd430b1 100644 --- a/vllm/entrypoints/generate/structured_decisions/serving.py +++ b/vllm/entrypoints/generate/structured_decisions/serving.py @@ -1,7 +1,6 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project import asyncio -import json from typing import Any from fastapi import Request @@ -24,10 +23,6 @@ from .strategies import DecisionLimits, ReadStrategy logger = init_logger(__name__) -def state_text(state: Any) -> str: - return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False) - - def parse_questions( request: StructuredDecisionRequest, limits: DecisionLimits, @@ -72,6 +67,18 @@ class ServingStructuredDecisions(BaseServing): self.strategy = strategy self.limits = strategy.limits() + def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]: + urls = { + "image": request.images, + "audio": request.audio, + "video": request.videos, + } + media = {modality: items for modality, items in urls.items() if items} + if unsupported := media.keys() - self.strategy.modalities: + names = ", ".join(sorted(unsupported)) + raise StructuredDecisionError(f"this model does not take {names} inputs") + return media + async def create_decision( self, request: StructuredDecisionRequest, @@ -87,8 +94,9 @@ class ServingStructuredDecisions(BaseServing): request_id = f"decision-{base_id}" try: questions = parse_questions(request, self.limits) + media = self._media(request) lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type] - engine_client.check_admission(len(questions)) + engine_client.check_admission(self.strategy.num_requests(questions)) trace_headers = ( None if raw_request is None @@ -97,13 +105,14 @@ class ServingStructuredDecisions(BaseServing): reads = await self.strategy.read( questions, request.instructions, - state_text(request.state), + self.strategy.render_state(request.state), request_id=request_id, chat_template_kwargs=request.chat_template_kwargs, lora_request=lora_request, priority=request.priority, cache_salt=request.cache_salt, trace_headers=trace_headers, + media=media, ) except StructuredDecisionError as e: return self.create_error_response(e) diff --git a/vllm/entrypoints/generate/structured_decisions/strategies.py b/vllm/entrypoints/generate/structured_decisions/strategies.py index 48debdb0b3..f4365924dd 100644 --- a/vllm/entrypoints/generate/structured_decisions/strategies.py +++ b/vllm/entrypoints/generate/structured_decisions/strategies.py @@ -5,24 +5,40 @@ Only models in READ_STRATEGIES are currently supported. """ +import asyncio import hashlib import json import math import random +import time from abc import ABC, abstractmethod from collections.abc import Mapping, Sequence from dataclasses import dataclass -from typing import Any +from functools import cached_property +from typing import Any, ClassVar, TypeVar +import numpy as np + +import vllm.envs as envs from vllm.config import ModelConfig from vllm.engine.protocol import EngineClient from vllm.entrypoints.generate.label_reads import next_token_label_reads -from vllm.inputs import EngineInput, tokens_input +from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input +from vllm.logger import init_logger from vllm.lora.request import LoRARequest +from vllm.multimodal.media import ( + MEDIA_CONNECTOR_REGISTRY, + ImageMediaIO, + MediaConnector, + MediaIO, + VideoMediaIO, +) +from vllm.pooling_params import PoolingParams from vllm.renderers.chat_utils import ChatTemplateContentFormatOption from vllm.renderers.inputs.preprocess import extract_prompt_components from vllm.renderers.online_renderer import OnlineRenderer from vllm.sampling_params import SamplingParams +from vllm.tasks import SupportedTask from vllm.tokenizers import TokenizerLike from .protocol import ReadPromptRequest @@ -34,6 +50,10 @@ from .question_types import ( label_softmax, ) +_M = TypeVar("_M") + +logger = init_logger(__name__) + @dataclass class ReadContext: @@ -59,13 +79,30 @@ class QuestionRead: output_tokens: int +def state_text(state: Any) -> str: + return state if isinstance(state, str) else json.dumps(state, ensure_ascii=False) + + class ReadStrategy(ABC): + #: The engine task reads run as. + task: ClassVar[SupportedTask] = "generate" + #: The modalities a request may attach media of. + modalities: ClassVar[frozenset[str]] = frozenset() + def __init__(self, context: ReadContext): self.context = context @abstractmethod def limits(self) -> DecisionLimits: ... + def render_state(self, state: Any) -> str: + """The request's state as the model reads it.""" + return state_text(state) + + def num_requests(self, questions: list[Question]) -> int: + """The engine requests a decision takes.""" + return len(questions) + @abstractmethod async def read( self, @@ -79,6 +116,7 @@ class ReadStrategy(ABC): priority: int, cache_salt: str | None = None, trace_headers: Mapping[str, str] | None = None, + media: Mapping[str, Sequence[str]] | None = None, ) -> list[QuestionRead]: """One read per question, in order.""" @@ -214,6 +252,7 @@ class NextTokenStrategy(ReadStrategy): priority: int, cache_salt: str | None = None, trace_headers: Mapping[str, str] | None = None, + media: Mapping[str, Sequence[str]] | None = None, ) -> list[QuestionRead]: ctx = self.context read_request = self._read_request(chat_template_kwargs, cache_salt) @@ -334,7 +373,315 @@ class DiffusionGemmaCanvasStrategy(NextTokenStrategy): ) +class JointSchemaStrategy(ReadStrategy): + """Clef-Omni. The whole request is one pooling read: the state, then every + question with its options, as the model was trained to read them. The + model's joint schema head scores each option, so a read has no labels. + + Mirrors ``encode_record`` in the checkpoint's ``joint_schema_model.py``. + Each piece is tokenized on its own, so the spans of the questions and the + options are known. The spans count from the end of the prompt, so they + hold after the media placeholders, which come before the state, expand.""" + + task = "token_classify" + modalities = frozenset({"image", "audio", "video"}) + + SYSTEM_PROMPT = ( + "Read the complete state and schema. Decide every field jointly. Each " + "answer must be exactly one of that field's allowed options." + ) + #: The reference's default bound on a prompt, in tokens. + MAX_LENGTH = 64000 + TYPE_IDS = {"noul": 0, "choice": 1, "score": 2} + NOUL_CRITERIA = { + "true": "The proposition is true or the answer is yes.", + "false": "The proposition is false or the answer is no.", + } + PLACEHOLDERS = { + "image": "<|vision_start|><|image_pad|><|vision_end|>\n", + "audio": "<|audio_start|><|audio_pad|><|audio_end|>\n", + "video": "<|vision_start|><|video_pad|><|vision_end|>\n", + } + IMAGE_SIZE = {"shortest_edge": 65_536, "longest_edge": 1_048_576} + #: The reference samples videos at 2 fps; the "clef_omni" video backend + #: reproduces the reference's sampling, pixel cap and frame count cap. + VIDEO_FPS = 2 + + def __init__(self, context: ReadContext): + super().__init__(context) + self.tokenizer = context.online_renderer.renderer.get_tokenizer() + self.max_length = min( + self.MAX_LENGTH, context.engine_client.model_config.max_model_len + ) + self.prefix = self._tokens( + f"<|im_start|>system\n{self.SYSTEM_PROMPT}<|im_end|>\n" + "<|im_start|>user\nSTATE:\n" + ) + self.suffix = self._tokens( + "\n<|im_end|>\n<|im_start|>assistant\n\n\n\n\n" + "JOINT SCHEMA DECISIONS:" + ) + + def limits(self) -> DecisionLimits: + return DecisionLimits(max_questions=64, max_options=len(LABELS)) + + def render_state(self, state: Any) -> str: + return self._render(state) + + def num_requests(self, questions: list[Question]) -> int: + return 1 + + @staticmethod + def _render(value: Any) -> str: + if isinstance(value, str): + return value + return json.dumps( + value, ensure_ascii=False, separators=(",", ":"), sort_keys=True + ) + + def _tokens(self, text: str) -> list[int]: + return self.tokenizer.encode(text, add_special_tokens=False) + + @cached_property + def _connector(self) -> MediaConnector: + model_config = self.context.engine_client.model_config + return MEDIA_CONNECTOR_REGISTRY.load( + envs.VLLM_MEDIA_CONNECTOR, + media_io_kwargs=model_config.get_multimodal_config().media_io_kwargs, + allowed_local_media_path=model_config.allowed_local_media_path, + allowed_media_domains=model_config.allowed_media_domains, + ) + + async def _fetch( + self, urls: Sequence[str], media_io: MediaIO[_M], timeout: int + ) -> list[_M]: + return await asyncio.gather( + *( + self._connector.load_from_url_async( + url, media_io, fetch_timeout=timeout + ) + for url in urls + ) + ) + + async def _soundtrack( + self, video_url: str + ) -> tuple[np.ndarray, int | float] | None: + """The video's soundtrack, or None when it has no decodable one.""" + try: + return await self._connector.fetch_audio_async(video_url) + except ValueError as e: + logger.debug("no decodable soundtrack in %s: %s", video_url, e) + return None + + async def _render_media( + self, media: Mapping[str, Sequence[str]], cache_salt: str | None + ) -> MultiModalInput: + """The system prompt and the media, with the media's placeholders + expanded. A video's soundtrack is heard with it when every video has + one.""" + connector = self._connector + video_io = VideoMediaIO( + ImageMediaIO(), + num_frames=-1, + fps=self.VIDEO_FPS, + video_backend="clef_omni", + ) + try: + images, audio, videos, tracks = await asyncio.gather( + asyncio.gather( + *map(connector.fetch_image_async, media.get("image", ())) + ), + asyncio.gather( + *map(connector.fetch_audio_async, media.get("audio", ())) + ), + self._fetch( + media.get("video", ()), video_io, envs.VLLM_VIDEO_FETCH_TIMEOUT + ), + asyncio.gather(*map(self._soundtrack, media.get("video", ()))), + ) + except ValueError as e: + raise StructuredDecisionError(f"media: {e}") from e + + audio_in_video = bool(videos) and all(t is not None for t in tracks) + if audio_in_video and audio: + raise StructuredDecisionError( + "audio clips cannot be read with videos that have soundtracks" + ) + text = ( + self.PLACEHOLDERS["image"] * len(images) + + self.PLACEHOLDERS["audio"] * len(audio) + + self.PLACEHOLDERS["video"] * len(videos) + ) + if audio_in_video: + audio = [track for track in tracks if track is not None] + mm_data: dict[str, Any] = {} + if images: + mm_data["image"] = images + if audio: + mm_data["audio"] = audio + if videos: + mm_data["video"] = videos + mm_kwargs: dict[str, Any] = {"images_kwargs": {"size": self.IMAGE_SIZE}} + if videos: + mm_kwargs["use_audio_in_video"] = audio_in_video + prompt = TokensPrompt( + prompt_token_ids=self.prefix + self._tokens(text), + multi_modal_data=mm_data, + mm_processor_kwargs=mm_kwargs, + ) + if cache_salt is not None: + prompt["cache_salt"] = cache_salt + renderer = self.context.online_renderer.renderer + try: + engine_input = await renderer.process_for_engine_async(prompt, time.time()) + except ValueError as e: + raise StructuredDecisionError(f"media: {e}") from e + assert engine_input["type"] == "multimodal" + return engine_input + + def _options(self, q: Question) -> list[tuple[int, str, Any]]: + """Each option's index in ``q.options``, id and description, in the + model's order: choices sorted by name, noul true then false, score + levels by index.""" + if q.type.name == "noul": + return [ + ( + i, + key, + self.NOUL_CRITERIA[key] if o.description is None else o.description, + ) + for i, (key, o) in enumerate(zip(("true", "false"), q.options)) + ] + if q.type.name == "choice": + return sorted( + ((i, o.name, o.description) for i, o in enumerate(q.options)), + key=lambda option: option[1], + ) + if q.type.name == "score": + return [(i, str(i), o.name) for i, o in enumerate(q.options)] + raise StructuredDecisionError( + f"question {q.id!r}: this model does not answer {q.type.name!r} questions" + ) + + def encode( + self, questions: list[Question], state: str, prefix: list[int] | None = None + ) -> tuple[list[int], dict[str, list[Any]], list[list[int]]]: + """The prompt, the schema for the pooler, and each question's option + indices in the order of the output. ``prefix`` is what precedes the + state: the system prompt, and the media if there are any.""" + prefix = self.prefix if prefix is None else prefix + schema_ids = self._tokens("\n\nSCHEMA FIELDS:\n") + question_spans, option_spans, orders = [], [], [] + for n, q in enumerate(questions, start=1): + schema_ids += self._tokens( + f"\nFIELD {n}\nID: {q.id}\nTYPE: {q.type.name}\nINSTRUCTION: " + ) + start = len(schema_ids) + schema_ids += self._tokens(q.instructions or q.id) + question_spans.append([start, len(schema_ids)]) + schema_ids += self._tokens("\nALLOWED OPTIONS:\n") + options = self._options(q) + for m, (_, option_id, description) in enumerate(options, start=1): + schema_ids += self._tokens(f"OPTION {m}: ") + start = len(schema_ids) + semantics = {"option_id": option_id} + if description is not None: + semantics["description"] = description + schema_ids += self._tokens(self._render(semantics)) + option_spans.append([start, len(schema_ids)]) + schema_ids += self._tokens("\n") + schema_ids += self._tokens("END FIELD\n") + orders.append([i for i, _, _ in options]) + + fixed = len(prefix) + len(schema_ids) + len(self.suffix) + if fixed > self.max_length: + raise StructuredDecisionError( + f"the questions take {fixed} tokens, more than the " + f"{self.max_length} this model reads" + ) + state_ids = self._tokens(state)[: self.max_length - fixed] + prompt = prefix + state_ids + schema_ids + self.suffix + # The schema ends len(self.suffix) tokens before the prompt does. + end = len(schema_ids) + len(self.suffix) + schema: dict[str, list[Any]] = { + "question_types": [self.TYPE_IDS[q.type.name] for q in questions], + "question_spans": [[s - end, e - end] for s, e in question_spans], + "option_spans": [[s - end, e - end] for s, e in option_spans], + "option_counts": [len(order) for order in orders], + } + return prompt, schema, orders + + async def read( + self, + questions: list[Question], + instructions: str | None, + state: str, + *, + request_id: str, + chat_template_kwargs: dict[str, Any] | None, + lora_request: LoRARequest | None, + priority: int, + cache_salt: str | None = None, + trace_headers: Mapping[str, str] | None = None, + media: Mapping[str, Sequence[str]] | None = None, + ) -> list[QuestionRead]: + if instructions: + raise StructuredDecisionError( + "this model reads a fixed prompt, so it takes no instructions" + ) + engine_input: EngineInput + if media: + engine_input = await self._render_media(media, cache_salt) + prompt, schema, orders = self.encode( + questions, state, engine_input["prompt_token_ids"] + ) + engine_input["prompt_token_ids"] = prompt + else: + prompt, schema, orders = self.encode(questions, state) + engine_input = tokens_input(prompt, cache_salt=cache_salt) + result = None + async for result in self.context.engine_client.encode( + engine_input, + PoolingParams(task="token_classify", extra_kwargs={"joint_schema": schema}), + f"{request_id}-0", + lora_request=lora_request, + trace_headers=trace_headers, + priority=priority, + ): + pass + if result is None: + raise ValueError("Failed to score the questions") + if result.error is not None: + raise ValueError(f"Failed to score the questions: {result.error.message}") + probs = result.outputs.data.tolist() + if len(probs) != sum(schema["option_counts"]): + raise ValueError( + f"Expected {sum(schema['option_counts'])} option scores, " + f"got {len(probs)}" + ) + + reads, offset = [], 0 + for n, order in enumerate(orders): + q_probs = [0.0] * len(order) + for i, p in zip(order, probs[offset : offset + len(order)]): + q_probs[i] = p + offset += len(order) + reads.append( + QuestionRead( + probs=q_probs, + label_mass=1.0, + argmax_is_label=True, + # One forward pass reads the prompt for every question. + input_tokens=len(prompt) if n == 0 else 0, + output_tokens=0, + ) + ) + return reads + + READ_STRATEGIES: dict[str, type[ReadStrategy]] = { + "ClefOmniForJointSchema": JointSchemaStrategy, "Qwen3ForCausalLM": NextTokenStrategy, "Qwen3_5ForConditionalGeneration": NextTokenStrategy, "Qwen3_5MoeForConditionalGeneration": NextTokenStrategy, @@ -347,7 +694,9 @@ READ_STRATEGIES: dict[str, type[ReadStrategy]] = { LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"}) -def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]: +def select_read_strategy( + model_config: ModelConfig, supported_tasks: Sequence[SupportedTask] +) -> type[ReadStrategy]: """Raise ValueError when the model cannot serve structured decisions.""" strategy = READ_STRATEGIES.get(model_config.architecture) if strategy is None: @@ -356,7 +705,12 @@ def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]: f"{model_config.architecture}. Supported architectures: " f"{sorted(READ_STRATEGIES)}" ) - if model_config.logprobs_mode not in LOGPROBS_MODES: + if strategy.task not in supported_tasks: + raise ValueError( + f"Structured decisions on {model_config.architecture} need the " + f"{strategy.task} task" + ) + if strategy.task == "generate" and model_config.logprobs_mode not in LOGPROBS_MODES: raise ValueError( "Structured decisions need --logprobs-mode raw_logprobs or " f"processed_logprobs, not {model_config.logprobs_mode}" diff --git a/vllm/entrypoints/launchers/api_server/app_state.py b/vllm/entrypoints/launchers/api_server/app_state.py index ce68890717..ff87dbea49 100644 --- a/vllm/entrypoints/launchers/api_server/app_state.py +++ b/vllm/entrypoints/launchers/api_server/app_state.py @@ -157,6 +157,19 @@ async def init_app_state( init_pooling_state(engine_client, state, args, request_logger, supported_tasks) + from vllm.entrypoints.generate.structured_decisions.api_router import ( + init_structured_decisions_state, + ) + + init_structured_decisions_state( + engine_client, + state, + args, + request_logger, + supported_tasks, + default_chat_template_kwargs, + ) + await init_endpoint_plugins_state(engine_client, state, args) state.enable_server_load_tracking = args.enable_server_load_tracking diff --git a/vllm/entrypoints/launchers/api_server/routers.py b/vllm/entrypoints/launchers/api_server/routers.py index 40d04862af..52c995a301 100644 --- a/vllm/entrypoints/launchers/api_server/routers.py +++ b/vllm/entrypoints/launchers/api_server/routers.py @@ -25,6 +25,12 @@ def register_api_routers( register_models_api_router(app) + from vllm.entrypoints.generate.structured_decisions.api_router import ( + register_structured_decisions_api_router, + ) + + register_structured_decisions_api_router(app) + from vllm.entrypoints.serve.sagemaker.api_router import ( attach_router as register_sagemaker_api_router, ) diff --git a/vllm/model_executor/model_loader/default_loader.py b/vllm/model_executor/model_loader/default_loader.py index b162f60b05..6bedabfdc0 100644 --- a/vllm/model_executor/model_loader/default_loader.py +++ b/vllm/model_executor/model_loader/default_loader.py @@ -247,9 +247,11 @@ class DefaultModelLoader(BaseModelLoader): subfolder=subfolder, revision=revision, ) - hf_weights_files = filter_duplicate_safetensors_files( - hf_weights_files, hf_folder, index_file - ) + # only filter the weights from index without explicit allow patterns + if allow_patterns_overrides is None: + hf_weights_files = filter_duplicate_safetensors_files( + hf_weights_files, hf_folder, index_file + ) else: hf_weights_files = filter_files_not_needed_for_inference(hf_weights_files) diff --git a/vllm/model_executor/models/clef_omni.py b/vllm/model_executor/models/clef_omni.py new file mode 100644 index 0000000000..7cd0d67687 --- /dev/null +++ b/vllm/model_executor/models/clef_omni.py @@ -0,0 +1,437 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Clef-Omni (https://huggingface.co/Cloudflare/clef-omni). + +The Qwen3-Omni thinker with a joint schema head that scores every option of +every question in one forward pass. The head reads the final hidden states of +the whole prompt, so it runs as a ``token_classify`` pooler. + +Each request carries its schema in ``PoolingParams.extra_kwargs`` under +``"joint_schema"``: + +- ``question_types``: 0 (noul), 1 (choice) or 2 (score) per question. +- ``question_spans``: the ``[start, end)`` of each question's instructions. +- ``option_spans``: the ``[start, end)`` of each option, question by question. +- ``option_counts``: the number of options of each question. + +Spans count from the end of the prompt (``-len(prompt) <= start < end <= 0``), +so they hold after multimodal placeholders are expanded. The output has one +value per option, in ``option_spans`` order: its probability within its +question, or its logit with ``use_activation=False``. +""" + +import math +from collections.abc import Callable, Mapping, Set +from dataclasses import dataclass +from typing import Any + +import numpy as np +import torch +import torch.nn.functional as F +from torch import nn +from transformers.feature_extraction_utils import BatchFeature + +from vllm.config import VllmConfig +from vllm.model_executor.layers.pooler import Pooler, PoolingParamsUpdate +from vllm.model_executor.layers.pooler.tokwise.methods import AllPool +from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding +from vllm.model_executor.model_loader.default_loader import DefaultModelLoader +from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.tasks import PoolingTask +from vllm.utils.torch_utils import async_tensor_h2d +from vllm.v1.outputs import PoolerOutput +from vllm.v1.pool.metadata import PoolingMetadata + +from .interfaces_base import default_pooling_type +from .qwen3_omni_moe_thinker import ( + Qwen3OmniMoeThinkerDummyInputsBuilder, + Qwen3OmniMoeThinkerForConditionalGeneration, + Qwen3OmniMoeThinkerMultiModalProcessor, + Qwen3OmniMoeThinkerProcessingInfo, +) + +JOINT_HEAD_WEIGHTS = "joint_head.safetensors" +NUM_QUESTION_TYPES = 3 +MAX_LOGIT_SCALE = math.log(100.0) + + +def segment_mean( + values: torch.Tensor, segment_ids: torch.Tensor, lengths: torch.Tensor +) -> torch.Tensor: + sums = values.new_zeros(lengths.shape[0], values.shape[-1], dtype=torch.float32) + sums.index_add_(0, segment_ids, values.float()) + return (sums / lengths[:, None]).to(values.dtype) + + +@dataclass +class SchemaIndices: + """Where one prompt's questions and options are, on the model's device.""" + + positions: torch.Tensor + """Every token of every span, questions first.""" + span_ids: torch.Tensor + """The span of each of ``positions``.""" + span_lengths: torch.Tensor + """``[Q + O]``.""" + question_types: torch.Tensor + """``[Q]``.""" + option_question: torch.Tensor + """The question of each option, ``[O]``.""" + option_slots: torch.Tensor + """Each option's flat index in a ``[Q, max_options]`` grid, ``[O]``.""" + option_mask: torch.Tensor + """``[Q, max_options]``, true at the slots of options.""" + + +class EvidenceRoutingLayer(nn.Module): + def __init__(self, width: int, heads: int, feedforward: int) -> None: + super().__init__() + self.query_norm = nn.LayerNorm(width) + self.memory_norm = nn.LayerNorm(width) + self.attention = nn.MultiheadAttention(width, heads, batch_first=True) + self.feedforward_norm = nn.LayerNorm(width) + # The Identity takes the place of the checkpoint's dropout. + self.feedforward = nn.Sequential( + nn.Linear(width, feedforward), + nn.GELU(), + nn.Identity(), + nn.Linear(feedforward, width), + ) + + def forward(self, queries: torch.Tensor, memory: torch.Tensor) -> torch.Tensor: + # Separate key and value tensors, like the reference, keep attention's + # unpacked input projections. + routed, _ = self.attention( + self.query_norm(queries), + self.memory_norm(memory), + self.memory_norm(memory), + need_weights=False, + ) + queries = queries + routed + return queries + self.feedforward(self.feedforward_norm(queries)) + + +class JointSchemaHead(nn.Module): + def __init__( + self, + hidden_size: int, + width: int, + routing_layers: int, + layers: int, + heads: int, + feedforward: int, + ) -> None: + super().__init__() + self.hidden_norm = nn.LayerNorm(hidden_size) + self.memory_projection = nn.Linear(hidden_size, width, bias=False) + self.question_projection = nn.Linear(hidden_size, width, bias=False) + self.option_question_projection = nn.Linear(hidden_size, width, bias=False) + self.global_projection = nn.Linear(hidden_size, width, bias=False) + self.option_context_projection = nn.Linear(hidden_size, width, bias=False) + self.option_lexical_projection = nn.Linear(hidden_size, width, bias=False) + self.type_embedding = nn.Embedding(NUM_QUESTION_TYPES, width) + self.evidence_layers = nn.ModuleList( + EvidenceRoutingLayer(width, heads, feedforward) + for _ in range(routing_layers) + ) + self.option_summary_norm = nn.LayerNorm(width) + self.layers = nn.ModuleList( + nn.TransformerDecoderLayer( + d_model=width, + nhead=heads, + dim_feedforward=feedforward, + dropout=0.0, + activation="gelu", + batch_first=True, + norm_first=True, + ) + for _ in range(layers) + ) + self.field_norm = nn.LayerNorm(width) + self.option_norm = nn.LayerNorm(width) + self.residual_scorer = nn.Sequential( + nn.Linear(width * 4, width), + nn.GELU(), + nn.Identity(), + nn.Linear(width, 1), + ) + self.prior_logit_scale = nn.Parameter(torch.zeros(())) + self.joint_logit_scale = nn.Parameter(torch.zeros(())) + self.residual_gate = nn.Parameter(torch.zeros(())) + + def forward( + self, + hidden_states: torch.Tensor, + lexical: torch.Tensor, + schema: SchemaIndices, + ) -> torch.Tensor: + """The ``[O]`` option logits of one prompt, from its ``[L, hidden]`` + final hidden states and each option's ``[O, hidden]`` mean output + embedding.""" + num_questions = schema.question_types.shape[0] + option_question = schema.option_question + hidden_states = self.hidden_norm(hidden_states) + memory = self.memory_projection(hidden_states).unsqueeze(0) + global_vector = hidden_states[-1] + means = segment_mean( + hidden_states[schema.positions], schema.span_ids, schema.span_lengths + ) + question_vectors, contexts = means[:num_questions], means[num_questions:] + + options = ( + self.option_context_projection(contexts) + + self.option_lexical_projection(lexical) + + self.option_question_projection(question_vectors)[option_question] + ).unsqueeze(0) + for layer in self.evidence_layers: + options = layer(options, memory) + options = options[0] + + # Each field summarizes its options, weighted by their match to it. + base_fields = self.question_projection(question_vectors) + width = options.shape[-1] + grid = options.new_zeros(schema.option_mask.numel(), width) + grid[schema.option_slots] = options + grid = grid.view(*schema.option_mask.shape, width) + routing = torch.einsum("qow,qw->qo", grid, base_fields) / math.sqrt(width) + routing = routing.masked_fill(~schema.option_mask, float("-inf")) + summaries = torch.einsum("qo,qow->qw", routing.softmax(dim=-1), grid) + fields = ( + base_fields + + self.option_summary_norm(summaries) + + self.global_projection(global_vector).unsqueeze(0) + + self.type_embedding(schema.question_types) + ).unsqueeze(0) + for layer in self.layers: + fields = layer(fields, memory) + fields = self.field_norm(fields[0])[option_question] + + anchors = F.normalize(question_vectors + global_vector, dim=-1) + prior_scale = self.prior_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp() + prior = prior_scale * ( + F.normalize(lexical, dim=-1) * anchors[option_question] + ).sum(dim=-1) + options = self.option_norm(options) + cosine = F.cosine_similarity(fields, options, dim=-1) + features = torch.cat( + [fields, options, fields * options, torch.abs(fields - options)], dim=-1 + ) + residual = self.residual_scorer(features).squeeze(-1) + joint_scale = self.joint_logit_scale.clamp(max=MAX_LOGIT_SCALE).exp() + joint = joint_scale * cosine + residual + return prior + torch.sigmoid(self.residual_gate) * joint + + +def parse_joint_schema( + extra_kwargs: dict[str, Any] | None, prompt_len: int +) -> tuple[list[int], list[tuple[int, int]], list[int]] | None: + """The question types, the absolute spans of the questions then of the + options, and the option counts. None if the schema is missing or invalid.""" + schema = (extra_kwargs or {}).get("joint_schema") + if not isinstance(schema, dict): + return None + try: + types = [int(t) for t in schema["question_types"]] + counts = [int(c) for c in schema["option_counts"]] + question_spans = [ + (int(start) + prompt_len, int(end) + prompt_len) + for start, end in schema["question_spans"] + ] + option_spans = [ + (int(start) + prompt_len, int(end) + prompt_len) + for start, end in schema["option_spans"] + ] + except (KeyError, TypeError, ValueError): + return None + spans = question_spans + option_spans + if ( + not types + or len(counts) != len(types) + or len(question_spans) != len(types) + or len(option_spans) != sum(counts) + or any(not 0 <= t < NUM_QUESTION_TYPES for t in types) + or any(c < 1 for c in counts) + or any(not 0 <= start < end <= prompt_len for start, end in spans) + ): + return None + return types, spans, counts + + +class JointSchemaPooler(Pooler): + """Runs the joint schema head on each finished prompt. + + It never raises, because it runs in the engine core: a prompt without a + valid schema, such as a profiling dummy, gets an empty output. + """ + + def __init__( + self, + head: JointSchemaHead, + embed_lexical: Callable[[torch.Tensor], torch.Tensor], + ) -> None: + super().__init__() + self.pooling = AllPool() + self.head = head + self.embed_lexical = embed_lexical + + def get_supported_tasks(self) -> Set[PoolingTask]: + return {"token_classify"} + + def get_pooling_updates(self, task: PoolingTask) -> PoolingParamsUpdate: + return PoolingParamsUpdate(requires_token_ids=True) + + def forward( + self, + hidden_states: torch.Tensor, + pooling_metadata: PoolingMetadata, + ) -> PoolerOutput: + pooled = self.pooling(hidden_states, pooling_metadata) + prompt_token_ids = pooling_metadata.get_prompt_token_ids_cpu() + outputs: list[torch.Tensor | None] = [] + for hidden, token_ids, params in zip( + pooled, prompt_token_ids, pooling_metadata.pooling_params + ): + if hidden is None: + outputs.append(None) + continue + parsed = parse_joint_schema(params.extra_kwargs, hidden.shape[0]) + if parsed is None: + outputs.append(hidden.new_empty(0, dtype=torch.float32)) + continue + schema, lexical_ids, lexical_span_ids = self._indices( + *parsed, hidden.device + ) + lexical = segment_mean( + self.embed_lexical( + async_tensor_h2d(token_ids[lexical_ids], hidden.device) + ), + lexical_span_ids, + schema.span_lengths[len(parsed[0]) :], + ) + logits = self.head(hidden, lexical, schema).float() + if params.use_activation is not False: + mask = schema.option_mask + grid = logits.new_full((mask.numel(),), float("-inf")) + grid[schema.option_slots] = logits + probs = grid.view(mask.shape).softmax(dim=-1) + logits = probs.view(-1)[schema.option_slots] + outputs.append(logits) + return outputs + + @staticmethod + def _indices( + types: list[int], + spans: list[tuple[int, int]], + counts: list[int], + device: torch.device, + ) -> tuple[SchemaIndices, torch.Tensor, torch.Tensor]: + """The schema's indices, plus the option tokens' positions (on the CPU) + and their option spans, for the lexical embeddings.""" + num_questions = len(types) + lengths = torch.tensor([end - start for start, end in spans]) + span_ids = torch.repeat_interleave(torch.arange(len(spans)), lengths) + positions = torch.cat([torch.arange(start, end) for start, end in spans]) + num_question_tokens = int(lengths[:num_questions].sum()) + + counts_cpu = torch.tensor(counts) + max_options = max(counts) + option_question = torch.repeat_interleave( + torch.arange(num_questions), counts_cpu + ) + option_slots = torch.cat( + [torch.arange(count) + q * max_options for q, count in enumerate(counts)] + ) + schema = SchemaIndices( + positions=async_tensor_h2d(positions, device), + span_ids=async_tensor_h2d(span_ids, device), + span_lengths=async_tensor_h2d(lengths, device), + question_types=async_tensor_h2d(types, device, dtype=torch.long), + option_question=async_tensor_h2d(option_question, device), + option_slots=async_tensor_h2d(option_slots, device), + option_mask=async_tensor_h2d( + torch.arange(max_options) < counts_cpu[:, None], device + ), + ) + lexical_span_ids = async_tensor_h2d( + span_ids[num_question_tokens:] - num_questions, device + ) + return schema, positions[num_question_tokens:], lexical_span_ids + + +MAX_FRAME_PIXELS = 256 * 32 * 32 + + +def cap_frame_pixels(frames: np.ndarray | torch.Tensor) -> np.ndarray | torch.Tensor: + """The reference reads each video frame at at most 256 * 32 * 32 pixels, + in even sizes. Frames are NHWC uint8, numpy or tensor; tensors stay on + their device (a CUDA tensor from torchcodec must not round-trip).""" + height, width = frames.shape[1:3] + if height * width <= MAX_FRAME_PIXELS: + return frames + scale = math.sqrt(MAX_FRAME_PIXELS / (width * height)) + new_width = max(2, round(width * scale / 2) * 2) + new_height = max(2, round(height * scale / 2) * 2) + is_tensor = isinstance(frames, torch.Tensor) + tensor = frames if is_tensor else torch.from_numpy(frames) + resized = F.interpolate( + tensor.permute(0, 3, 1, 2).float(), + size=(new_height, new_width), + mode="bilinear", + ) + if tensor.dtype == torch.uint8: + resized = resized.round() + resized = resized.to(tensor.dtype).permute(0, 2, 3, 1).contiguous() + return resized if is_tensor else resized.numpy() + + +def _cap_video(item: Any) -> Any: + if isinstance(item, tuple): + frames, metadata = item + return cap_frame_pixels(frames), metadata + return cap_frame_pixels(item) + + +class ClefOmniMultiModalProcessor(Qwen3OmniMoeThinkerMultiModalProcessor): + """Caps video frames at the reference's per-frame pixel count.""" + + def _call_hf_processor( + self, + hf_data: Mapping[str, object], + hf_kwargs: Mapping[str, object], + ) -> BatchFeature: + if videos := hf_data.get("videos"): + assert isinstance(videos, list) + hf_data = {**hf_data, "videos": [_cap_video(v) for v in videos]} + return super()._call_hf_processor(hf_data, hf_kwargs) + + +@default_pooling_type(tok_pooling_type="ALL") +@MULTIMODAL_REGISTRY.register_processor( + ClefOmniMultiModalProcessor, + info=Qwen3OmniMoeThinkerProcessingInfo, + dummy_inputs=Qwen3OmniMoeThinkerDummyInputsBuilder, +) +class ClefOmniForJointSchema(Qwen3OmniMoeThinkerForConditionalGeneration): + is_pooling_model = True + + def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): + super().__init__(vllm_config=vllm_config, prefix=prefix) + model_config = vllm_config.model_config + head_config = model_config.hf_config.joint_head_config + self.pooler = JointSchemaPooler( + JointSchemaHead(**head_config), self._embed_lexical + ) + # The head's weights ship as their own safetensors file. + self.secondary_weights = [ + DefaultModelLoader.Source( + model_or_path=model_config.model, + revision=model_config.revision, + prefix="pooler.head.", + allow_patterns_overrides=[JOINT_HEAD_WEIGHTS], + ) + ] + + def _embed_lexical(self, token_ids: torch.Tensor) -> torch.Tensor: + # The head embeds options with the output embeddings. ParallelLMHead + # forbids lookups, so use the vocab-parallel embedding's. + return VocabParallelEmbedding.forward(self.language_model.lm_head, token_ids) diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index 4ae80a4a36..321b5c2289 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -83,6 +83,7 @@ from vllm.multimodal.processing.processor import ( PromptUpdate, PromptUpdateDetails, ) +from vllm.multimodal.utils import copy_mm_embedding_modality from vllm.sequence import IntermediateTensors from vllm.transformers_utils.processor import cached_processor_from_config from vllm.utils.gpu_sync_debug import gpu_sync_allowed @@ -1498,8 +1499,12 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( audio_in_video_item_idx += 1 + # Match the fps the processor used, as HF and the mrope positions do. + processed_second_per_grid_ts = out_mm_data.get("second_per_grid_ts") second_per_grid_ts = hf_processor_mm_kwargs.get("second_per_grid_ts", None) - if second_per_grid_ts: + if isinstance(processed_second_per_grid_ts, torch.Tensor): + video_second_per_grid_t = processed_second_per_grid_ts[item_idx].item() + elif second_per_grid_ts: video_second_per_grid_t = second_per_grid_ts[item_idx] else: video_second_per_grid_t = 2.0 @@ -1904,7 +1909,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( embeddings_main, embeddings_multiscale = torch.split( embeddings, [visual_dim, multi_dim], dim=-1 ) - multimodal_embeddings[index] = embeddings_main + multimodal_embeddings[index] = copy_mm_embedding_modality( + embeddings, embeddings_main + ) multimodal_embeddings_multiscale.append(embeddings_multiscale) if not is_interleaved: current_positions = mm_positions[ diff --git a/vllm/model_executor/models/registry.py b/vllm/model_executor/models/registry.py index 427648045f..36f9bd6e6f 100644 --- a/vllm/model_executor/models/registry.py +++ b/vllm/model_executor/models/registry.py @@ -287,6 +287,7 @@ _REWARD_MODELS = { _TOKEN_CLASSIFICATION_MODELS = { "BertForTokenClassification": ("bert", "BertForTokenClassification"), + "ClefOmniForJointSchema": ("clef_omni", "ClefOmniForJointSchema"), "ModernBertForTokenClassification": ( "modernbert", "ModernBertForTokenClassification", diff --git a/vllm/multimodal/video.py b/vllm/multimodal/video.py index fc0cdc28d4..75a6c1beb1 100644 --- a/vllm/multimodal/video.py +++ b/vllm/multimodal/video.py @@ -1570,3 +1570,67 @@ class OpenCVDynamicOpenPanguVideoBackend(VideoLoader): valid_frame_indices=valid_frame_indices, ) return frames, metadata + + +@VIDEO_LOADER_REGISTRY.register("clef_omni") +class ClefOmniVideoBackend(VideoBackend): + """Clef-Omni's reference sampling: the first frame at or after each + ``1/fps`` boundary (frame i sits at ``i / original_fps`` seconds, so the + k-th boundary is crossed by frame ``ceil(k * original_fps / fps)``), + capped at 768 frames and padded to an even count by repeating the last + frame.""" + + MAX_FRAMES = 768 + + @classmethod + def compute_frames_index_to_sample( + cls, + source: VideoSourceMetadata, + target: VideoTargetMetadata, + **kwargs, + ) -> list[int]: + if source.original_fps <= 0: + raise ValueError("video has an unknown frame rate") + step = source.original_fps / target.fps + indices = [] + k = 0 + while (index := math.ceil(k * step - 1e-6)) < source.total_frames_num: + indices.append(index) + if len(indices) > cls.MAX_FRAMES: + raise ValueError( + f"video is longer than {cls.MAX_FRAMES / target.fps:.0f} seconds" + ) + k += 1 + if not indices: + raise ValueError("video has no decodable frames") + return indices + + @classmethod + def load_bytes( + cls, + data: bytes, + num_frames: int = -1, + fps: int = 2, + max_duration: int = 300, + frame_recovery: bool = False, + *, + backend: VideoDecoderBackend = "opencv", + **kwargs: Any, + ) -> tuple[DecodedFrames, dict[str, Any]]: + frames, metadata = super().load_bytes( + data, + num_frames=num_frames, + fps=fps, + max_duration=max_duration, + frame_recovery=frame_recovery, + backend=backend, + **kwargs, + ) + if len(frames) % 2: + last = frames[-1:] + if isinstance(frames, torch.Tensor): + frames = torch.cat([frames, last]) + else: + frames = np.concatenate([frames, last]) + metadata["frames_indices"].append(metadata["frames_indices"][-1]) + return frames, metadata diff --git a/vllm/transformers_utils/model_arch_config_convertor.py b/vllm/transformers_utils/model_arch_config_convertor.py index d79c1275b6..147b65759d 100644 --- a/vllm/transformers_utils/model_arch_config_convertor.py +++ b/vllm/transformers_utils/model_arch_config_convertor.py @@ -17,6 +17,10 @@ from vllm.transformers_utils.config import ( ConfigFormat, get_safetensors_params_metadata, ) +from vllm.transformers_utils.repo_utils import ( + file_or_path_exists, + get_hf_file_to_dict, +) from vllm.utils.torch_utils import common_broadcastable_dtype logger = init_logger(__name__) @@ -643,6 +647,31 @@ class MimoV2ModelArchConfigConvertor(ModelArchConfigConvertorBase): _strip_mimo_v2_attention_chunk_size(hf_config, hf_text_config) +class Qwen3OmniMoeModelArchConfigConvertor(ModelArchConfigConvertorBase): + def __init__( + self, + hf_config: PreTrainedConfig, + hf_text_config: PreTrainedConfig, + revision: str | None = None, + ): + # Clef-Omni keeps Qwen3-Omni's config and adds a joint schema head; + # route to its wrapper class when the head's config file is present. + # Mutate hf_config (like the deepseek_v4/mimo_v2 rewrites above) + # because model-class resolution reads hf_config.architectures. + if getattr(hf_config, "architectures", None) == [ + "Qwen3OmniMoeForConditionalGeneration" + ]: + model_path = hf_config.name_or_path + if model_path and file_or_path_exists( + model_path, "joint_head_config.json", revision + ): + hf_config.architectures = ["ClefOmniForJointSchema"] + hf_config.joint_head_config = get_hf_file_to_dict( + "joint_head_config.json", model_path, revision + ) + super().__init__(hf_config, hf_text_config, revision) + + class MimoV2MTPModelArchConfigConvertor(ModelArchConfigConvertorBase): def __init__( self, @@ -822,6 +851,7 @@ MODEL_ARCH_CONFIG_CONVERTORS = { "bailing_hybrid_v3_mtp": BailingHybridV3MTPModelArchConfigConvertor, "pangu_ultra_moe_mtp": PanguUltraMoeMTPModelArchConfigConvertor, "qwen3_5_mtp": Qwen3_5MTPModelArchConfigConvertor, + "qwen3_omni_moe": Qwen3OmniMoeModelArchConfigConvertor, "qwen3_next_mtp": Qwen3NextMTPModelArchConfigConvertor, "qwen4_exp_mtp": Qwen4ExpMTPModelArchConfigConvertor, "RefinedWeb": FalconModelArchConfigConvertor,