[Model] Support Cloudflare/clef-omni on /v1/systemone (#61064)

Signed-off-by: zjy0516 <riverclouds.zhu@qq.com>
Signed-off-by: Isotr0py <Isotr0py@outlook.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
Jiangyun Zhu
2026-10-12 21:27:34 +08:00
committed by GitHub
co-authored by Claude Opus 5.5 Isotr0py
parent 314ab8cffd
commit 092c35dd15
20 changed files with 1592 additions and 69 deletions
@@ -1,12 +1,18 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import json
from io import BytesIO
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
import numpy as np
import pybase64 as base64
import pytest
import torch
from fastapi import FastAPI, Request
from fastapi.testclient import TestClient
from PIL import Image
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
@@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import (
)
from vllm.entrypoints.generate.structured_decisions.serving import (
ServingStructuredDecisions,
state_text,
parse_questions,
)
from vllm.entrypoints.generate.structured_decisions.strategies import (
DiffusionGemmaCanvasStrategy,
JointSchemaStrategy,
NextTokenStrategy,
ReadContext,
reply_label_ids,
select_read_strategy,
state_text,
)
from vllm.entrypoints.openai.models.protocol import BaseModelPath
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.register import (
init_exception_handler,
)
from vllm.inputs import mm_input
from vllm.logprobs import Logprob
from vllm.outputs import CompletionOutput, RequestOutput
from vllm.multimodal.media import MediaConnector, MediaWithBytes
from vllm.outputs import (
CompletionOutput,
PoolingOutput,
PoolingRequestOutput,
RequestOutput,
)
from vllm.renderers.hf import HfRenderer
from vllm.renderers.online_renderer import OnlineRenderer
@@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
def test_strategy_selection():
qwen = "Qwen3ForCausalLM"
assert select_read_strategy(model(qwen)) is NextTokenStrategy
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
qwen, generate = "Qwen3ForCausalLM", ("generate",)
assert select_read_strategy(model(qwen), generate) is NextTokenStrategy
assert (
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"))
select_read_strategy(model(qwen, "processed_logprobs"), generate)
is NextTokenStrategy
)
assert (
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate)
is DiffusionGemmaCanvasStrategy
)
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
select_read_strategy(model("LlamaForCausalLM"))
select_read_strategy(model("LlamaForCausalLM"), generate)
with pytest.raises(ValueError, match="not raw_logits"):
select_read_strategy(model(qwen, "raw_logits"))
select_read_strategy(model(qwen, "raw_logits"), generate)
# A generative architecture served as a pooling model has no read strategy.
with pytest.raises(ValueError, match="need the generate task"):
select_read_strategy(model(qwen), ("embed",))
# Clef-Omni scores options with its head, so it reads no logprobs.
clef = model("ClefOmniForJointSchema", "raw_logits")
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
def test_route_is_registered_by_default():
@@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen):
@pytest.fixture
def serving_and_engine(qwen):
def engine_and_renderer(qwen):
tokenizer, _ = qwen
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
renderer = HfRenderer(
@@ -148,6 +173,19 @@ def serving_and_engine(qwen):
chat_template=None,
chat_template_content_format="auto",
)
yield engine, online_renderer
renderer.shutdown()
def serving(engine, strategy) -> ServingStructuredDecisions:
name = engine.model_config.model
models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)])
return ServingStructuredDecisions(models, strategy)
@pytest.fixture
def serving_and_engine(engine_and_renderer):
engine, online_renderer = engine_and_renderer
async def generate(prompt, params, request_id, **kwargs):
label_ids = params.logprob_token_ids
@@ -171,11 +209,7 @@ def serving_and_engine(qwen):
engine.generate.side_effect = generate
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
models = OpenAIServingModels(
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
)
yield ServingStructuredDecisions(models, strategy), engine
renderer.shutdown()
return serving(engine, strategy), engine
@pytest.mark.asyncio
@@ -256,3 +290,286 @@ def test_canvas_read():
strategy._read_input(
{"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4]
)
JOINT_SCHEMA_PROMPT = """<|im_start|>system
Read the complete state and schema. Decide every field jointly. Each answer \
must be exactly one of that field's allowed options.<|im_end|>
<|im_start|>user
STATE:
{"customer":{"tier":"gold"},"ticket":"Charged twice"}
SCHEMA FIELDS:
FIELD 1
ID: team
TYPE: choice
INSTRUCTION: Which team?
ALLOWED OPTIONS:
OPTION 1: {"description":"payments","option_id":"billing"}
OPTION 2: {"option_id":"security"}
OPTION 3: {"description":"deliveries","option_id":"shipping"}
END FIELD
FIELD 2
ID: urgent
TYPE: noul
INSTRUCTION: urgent
ALLOWED OPTIONS:
OPTION 1: {"description":"money is moving","option_id":"true"}
OPTION 2: {"description":"The proposition is false or the answer is no.",\
"option_id":"false"}
END FIELD
FIELD 3
ID: tone
TYPE: score
INSTRUCTION: How upset?
ALLOWED OPTIONS:
OPTION 1: {"description":"calm","option_id":"0"}
OPTION 2: {"description":"annoyed","option_id":"1"}
OPTION 3: {"description":"furious","option_id":"2"}
END FIELD
<|im_end|>
<|im_start|>assistant
<think>
</think>
JOINT SCHEMA DECISIONS:"""
JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
state={"ticket": "Charged twice", "customer": {"tier": "gold"}},
questions={
"team": {
"type": "choice",
"instructions": "Which team?",
"criteria": {
"shipping": "deliveries",
"billing": "payments",
"security": None,
},
},
"urgent": {"type": "noul", "criteria": {"true": "money is moving"}},
"tone": {
"type": "score",
"instructions": "How upset?",
"criteria": ["calm", "annoyed", "furious"],
},
},
)
@pytest.fixture
def joint_schema(qwen, engine_and_renderer):
tokenizer, _ = qwen
engine, online_renderer = engine_and_renderer
# The probability the head gives each option, by option id.
scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2}
scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3}
async def encode(prompt, params, request_id, **kwargs):
ids = prompt["prompt_token_ids"]
spans = params.extra_kwargs["joint_schema"]["option_spans"]
options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans]
data = torch.tensor([scores[o["option_id"]] for o in options])
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
engine.encode.side_effect = encode
strategy = JointSchemaStrategy(
ReadContext(engine, online_renderer, None, "auto", {})
)
return serving(engine, strategy), engine, tokenizer
@pytest.mark.asyncio
async def test_joint_schema_reads_every_question_at_once(joint_schema):
serving, engine, tokenizer = joint_schema
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt")
)
assert isinstance(response, StructuredDecisionResponse)
engine.encode.assert_called_once()
engine.check_admission.assert_called_once_with(1)
prompt, params = engine.encode.call_args.args[:2]
ids = prompt["prompt_token_ids"]
assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT
assert prompt["cache_salt"] == "salt"
assert params.task == "token_classify"
schema = params.extra_kwargs["joint_schema"]
assert schema["question_types"] == [1, 0, 2]
assert schema["option_counts"] == [3, 2, 3]
assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [
"Which team?",
"urgent",
"How upset?",
]
# Options are read in the model's order and answered in the request's.
team = response.answers["team"]
assert team["choice"] == "billing"
assert list(team["probabilities"]) == ["shipping", "billing", "security"]
assert team["probabilities"] == pytest.approx(
{"shipping": 0.2, "billing": 0.7, "security": 0.1}
)
assert response.answers["urgent"]["noul"] == pytest.approx(0.9)
assert response.answers["tone"]["score"] == pytest.approx(1.1)
assert response.usage.input_tokens == len(ids)
assert response.usage.output_tokens == 0
assert all(d.label_mass == 1.0 for d in response.diagnostics.values())
@pytest.mark.asyncio
async def test_joint_schema_rejects_instructions(joint_schema):
serving, engine, _ = joint_schema
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.")
)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
engine.encode.assert_not_called()
def test_joint_schema_truncates_the_state(joint_schema):
serving, _, _ = joint_schema
strategy = serving.strategy
request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST)
questions = parse_questions(request, strategy.limits())
prompt, schema, _ = strategy.encode(questions, "word " * 100)
strategy.max_length = len(prompt) - 50
truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100)
# The state loses 50 tokens and the questions stay at the end.
assert len(truncated) == strategy.max_length
assert truncated[-200:] == prompt[-200:]
assert truncated_schema == schema
strategy.max_length = 100
with pytest.raises(StructuredDecisionError, match="questions take"):
strategy.encode(questions, "word")
@pytest.mark.asyncio
async def test_media_needs_a_model_that_takes_it(serving_and_engine):
serving, engine = serving_and_engine
request = StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"]
)
response = await serving.create_decision(request)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400
assert "does not take image inputs" in response.error.message
engine.generate.assert_not_called()
@pytest.mark.asyncio
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
strategy.__dict__["_connector"] = MediaConnector()
image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
rendered: dict[str, Any] = {}
async def process_for_engine_async(prompt, arrival_time):
rendered.update(prompt)
# The image's placeholder expands to three tokens.
ids = [
token
for pad in prompt["prompt_token_ids"]
for token in ([pad] * 3 if pad == image_pad else [pad])
]
return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt"))
renderer = strategy.context.online_renderer.renderer
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
buffer = BytesIO()
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
response = await serving.create_decision(
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
)
assert isinstance(response, StructuredDecisionResponse)
placeholder = tokenizer.encode(
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
)
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE}
}
prompt = engine.encode.call_args.args[0]
assert prompt["type"] == "multimodal"
ids = prompt["prompt_token_ids"]
expanded = len(strategy.prefix) + len(placeholder) + 2
assert ids[:expanded].count(image_pad) == 3
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
assert tokenizer.decode(ids[expanded:]) == state_and_questions
assert response.answers["team"]["choice"] == "billing"
assert response.usage.input_tokens == len(ids)
@pytest.mark.asyncio
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
serving, engine, tokenizer = joint_schema
strategy = serving.strategy
video = MediaWithBytes(
(np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes"
)
track = (np.zeros(160, np.float32), 16000)
rendered: dict[str, Any] = {}
class FakeConnector:
async def fetch_image_async(self, url):
raise AssertionError("no images in this test")
async def load_from_url_async(self, url, media_io, *, fetch_timeout=None):
return video
async def fetch_audio_async(self, url):
if url == "https://example.org/silent.mp4":
raise ValueError("No audio stream found.")
return track
strategy.__dict__["_connector"] = FakeConnector()
async def process_for_engine_async(prompt, arrival_time):
rendered.clear()
rendered.update(prompt)
return mm_input(prompt["prompt_token_ids"], {}, {}, {})
renderer = strategy.context.online_renderer.renderer
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"]
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
assert rendered["mm_processor_kwargs"] == {
"images_kwargs": {"size": strategy.IMAGE_SIZE},
"use_audio_in_video": True,
}
# A video without a decodable soundtrack is watched silently.
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"]
)
)
assert isinstance(response, StructuredDecisionResponse)
assert rendered["multi_modal_data"] == {"video": [video]}
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
# Audio clips cannot be read with videos that have soundtracks.
response = await serving.create_decision(
StructuredDecisionRequest(
**JOINT_SCHEMA_REQUEST,
videos=["https://example.org/talking.mp4"],
audio=["https://example.org/clip.wav"],
)
)
assert isinstance(response, ErrorResponse)
assert response.error.code == 400