[Model] Support Cloudflare/clef-omni on /v1/systemone (#61064)
Signed-off-by: zjy0516 <riverclouds.zhu@qq.com> Signed-off-by: Isotr0py <Isotr0py@outlook.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Isotr0py <Isotr0py@outlook.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Isotr0py
parent
314ab8cffd
commit
092c35dd15
@@ -1,12 +1,18 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import numpy as np
|
||||
import pybase64 as base64
|
||||
import pytest
|
||||
import torch
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -23,22 +29,32 @@ from vllm.entrypoints.generate.structured_decisions.question_types import (
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.serving import (
|
||||
ServingStructuredDecisions,
|
||||
state_text,
|
||||
parse_questions,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
DiffusionGemmaCanvasStrategy,
|
||||
JointSchemaStrategy,
|
||||
NextTokenStrategy,
|
||||
ReadContext,
|
||||
reply_label_ids,
|
||||
select_read_strategy,
|
||||
state_text,
|
||||
)
|
||||
from vllm.entrypoints.openai.models.protocol import BaseModelPath
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.register import (
|
||||
init_exception_handler,
|
||||
)
|
||||
from vllm.inputs import mm_input
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.outputs import CompletionOutput, RequestOutput
|
||||
from vllm.multimodal.media import MediaConnector, MediaWithBytes
|
||||
from vllm.outputs import (
|
||||
CompletionOutput,
|
||||
PoolingOutput,
|
||||
PoolingRequestOutput,
|
||||
RequestOutput,
|
||||
)
|
||||
from vllm.renderers.hf import HfRenderer
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
|
||||
@@ -53,17 +69,26 @@ def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
|
||||
|
||||
|
||||
def test_strategy_selection():
|
||||
qwen = "Qwen3ForCausalLM"
|
||||
assert select_read_strategy(model(qwen)) is NextTokenStrategy
|
||||
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
|
||||
qwen, generate = "Qwen3ForCausalLM", ("generate",)
|
||||
assert select_read_strategy(model(qwen), generate) is NextTokenStrategy
|
||||
assert (
|
||||
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"))
|
||||
select_read_strategy(model(qwen, "processed_logprobs"), generate)
|
||||
is NextTokenStrategy
|
||||
)
|
||||
assert (
|
||||
select_read_strategy(model("DiffusionGemmaForBlockDiffusion"), generate)
|
||||
is DiffusionGemmaCanvasStrategy
|
||||
)
|
||||
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
|
||||
select_read_strategy(model("LlamaForCausalLM"))
|
||||
select_read_strategy(model("LlamaForCausalLM"), generate)
|
||||
with pytest.raises(ValueError, match="not raw_logits"):
|
||||
select_read_strategy(model(qwen, "raw_logits"))
|
||||
select_read_strategy(model(qwen, "raw_logits"), generate)
|
||||
# A generative architecture served as a pooling model has no read strategy.
|
||||
with pytest.raises(ValueError, match="need the generate task"):
|
||||
select_read_strategy(model(qwen), ("embed",))
|
||||
# Clef-Omni scores options with its head, so it reads no logprobs.
|
||||
clef = model("ClefOmniForJointSchema", "raw_logits")
|
||||
assert select_read_strategy(clef, ("token_classify",)) is JointSchemaStrategy
|
||||
|
||||
|
||||
def test_route_is_registered_by_default():
|
||||
@@ -126,7 +151,7 @@ def test_labels_start_the_reply(qwen):
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serving_and_engine(qwen):
|
||||
def engine_and_renderer(qwen):
|
||||
tokenizer, _ = qwen
|
||||
model_config = ModelConfig(model="Qwen/Qwen3-0.6B", dtype="float32")
|
||||
renderer = HfRenderer(
|
||||
@@ -148,6 +173,19 @@ def serving_and_engine(qwen):
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
yield engine, online_renderer
|
||||
renderer.shutdown()
|
||||
|
||||
|
||||
def serving(engine, strategy) -> ServingStructuredDecisions:
|
||||
name = engine.model_config.model
|
||||
models = OpenAIServingModels(engine, [BaseModelPath(name=name, model_path=name)])
|
||||
return ServingStructuredDecisions(models, strategy)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def serving_and_engine(engine_and_renderer):
|
||||
engine, online_renderer = engine_and_renderer
|
||||
|
||||
async def generate(prompt, params, request_id, **kwargs):
|
||||
label_ids = params.logprob_token_ids
|
||||
@@ -171,11 +209,7 @@ def serving_and_engine(qwen):
|
||||
|
||||
engine.generate.side_effect = generate
|
||||
strategy = NextTokenStrategy(ReadContext(engine, online_renderer, None, "auto", {}))
|
||||
models = OpenAIServingModels(
|
||||
engine, [BaseModelPath(name=model_config.model, model_path=model_config.model)]
|
||||
)
|
||||
yield ServingStructuredDecisions(models, strategy), engine
|
||||
renderer.shutdown()
|
||||
return serving(engine, strategy), engine
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@@ -256,3 +290,286 @@ def test_canvas_read():
|
||||
strategy._read_input(
|
||||
{"type": "token", "prompt_token_ids": [1, 2, 3, 4]}, [1, 2, 3, 4]
|
||||
)
|
||||
|
||||
|
||||
JOINT_SCHEMA_PROMPT = """<|im_start|>system
|
||||
Read the complete state and schema. Decide every field jointly. Each answer \
|
||||
must be exactly one of that field's allowed options.<|im_end|>
|
||||
<|im_start|>user
|
||||
STATE:
|
||||
{"customer":{"tier":"gold"},"ticket":"Charged twice"}
|
||||
|
||||
SCHEMA FIELDS:
|
||||
|
||||
FIELD 1
|
||||
ID: team
|
||||
TYPE: choice
|
||||
INSTRUCTION: Which team?
|
||||
ALLOWED OPTIONS:
|
||||
OPTION 1: {"description":"payments","option_id":"billing"}
|
||||
OPTION 2: {"option_id":"security"}
|
||||
OPTION 3: {"description":"deliveries","option_id":"shipping"}
|
||||
END FIELD
|
||||
|
||||
FIELD 2
|
||||
ID: urgent
|
||||
TYPE: noul
|
||||
INSTRUCTION: urgent
|
||||
ALLOWED OPTIONS:
|
||||
OPTION 1: {"description":"money is moving","option_id":"true"}
|
||||
OPTION 2: {"description":"The proposition is false or the answer is no.",\
|
||||
"option_id":"false"}
|
||||
END FIELD
|
||||
|
||||
FIELD 3
|
||||
ID: tone
|
||||
TYPE: score
|
||||
INSTRUCTION: How upset?
|
||||
ALLOWED OPTIONS:
|
||||
OPTION 1: {"description":"calm","option_id":"0"}
|
||||
OPTION 2: {"description":"annoyed","option_id":"1"}
|
||||
OPTION 3: {"description":"furious","option_id":"2"}
|
||||
END FIELD
|
||||
|
||||
<|im_end|>
|
||||
<|im_start|>assistant
|
||||
<think>
|
||||
|
||||
</think>
|
||||
|
||||
JOINT SCHEMA DECISIONS:"""
|
||||
|
||||
JOINT_SCHEMA_REQUEST: dict[str, Any] = dict(
|
||||
state={"ticket": "Charged twice", "customer": {"tier": "gold"}},
|
||||
questions={
|
||||
"team": {
|
||||
"type": "choice",
|
||||
"instructions": "Which team?",
|
||||
"criteria": {
|
||||
"shipping": "deliveries",
|
||||
"billing": "payments",
|
||||
"security": None,
|
||||
},
|
||||
},
|
||||
"urgent": {"type": "noul", "criteria": {"true": "money is moving"}},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset?",
|
||||
"criteria": ["calm", "annoyed", "furious"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def joint_schema(qwen, engine_and_renderer):
|
||||
tokenizer, _ = qwen
|
||||
engine, online_renderer = engine_and_renderer
|
||||
# The probability the head gives each option, by option id.
|
||||
scores = {"billing": 0.7, "security": 0.1, "shipping": 0.2}
|
||||
scores |= {"true": 0.9, "false": 0.1, "0": 0.2, "1": 0.5, "2": 0.3}
|
||||
|
||||
async def encode(prompt, params, request_id, **kwargs):
|
||||
ids = prompt["prompt_token_ids"]
|
||||
spans = params.extra_kwargs["joint_schema"]["option_spans"]
|
||||
options = [json.loads(tokenizer.decode(ids[s:e])) for s, e in spans]
|
||||
data = torch.tensor([scores[o["option_id"]] for o in options])
|
||||
yield PoolingRequestOutput(request_id, PoolingOutput(data), ids, 0, True)
|
||||
|
||||
engine.encode.side_effect = encode
|
||||
strategy = JointSchemaStrategy(
|
||||
ReadContext(engine, online_renderer, None, "auto", {})
|
||||
)
|
||||
return serving(engine, strategy), engine, tokenizer
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_reads_every_question_at_once(joint_schema):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, cache_salt="salt")
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
|
||||
engine.encode.assert_called_once()
|
||||
engine.check_admission.assert_called_once_with(1)
|
||||
prompt, params = engine.encode.call_args.args[:2]
|
||||
ids = prompt["prompt_token_ids"]
|
||||
assert tokenizer.decode(ids) == JOINT_SCHEMA_PROMPT
|
||||
assert prompt["cache_salt"] == "salt"
|
||||
assert params.task == "token_classify"
|
||||
schema = params.extra_kwargs["joint_schema"]
|
||||
assert schema["question_types"] == [1, 0, 2]
|
||||
assert schema["option_counts"] == [3, 2, 3]
|
||||
assert [tokenizer.decode(ids[s:e]) for s, e in schema["question_spans"]] == [
|
||||
"Which team?",
|
||||
"urgent",
|
||||
"How upset?",
|
||||
]
|
||||
|
||||
# Options are read in the model's order and answered in the request's.
|
||||
team = response.answers["team"]
|
||||
assert team["choice"] == "billing"
|
||||
assert list(team["probabilities"]) == ["shipping", "billing", "security"]
|
||||
assert team["probabilities"] == pytest.approx(
|
||||
{"shipping": 0.2, "billing": 0.7, "security": 0.1}
|
||||
)
|
||||
assert response.answers["urgent"]["noul"] == pytest.approx(0.9)
|
||||
assert response.answers["tone"]["score"] == pytest.approx(1.1)
|
||||
assert response.usage.input_tokens == len(ids)
|
||||
assert response.usage.output_tokens == 0
|
||||
assert all(d.label_mass == 1.0 for d in response.diagnostics.values())
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_rejects_instructions(joint_schema):
|
||||
serving, engine, _ = joint_schema
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, instructions="Be strict.")
|
||||
)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
engine.encode.assert_not_called()
|
||||
|
||||
|
||||
def test_joint_schema_truncates_the_state(joint_schema):
|
||||
serving, _, _ = joint_schema
|
||||
strategy = serving.strategy
|
||||
request = StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST)
|
||||
questions = parse_questions(request, strategy.limits())
|
||||
prompt, schema, _ = strategy.encode(questions, "word " * 100)
|
||||
strategy.max_length = len(prompt) - 50
|
||||
truncated, truncated_schema, _ = strategy.encode(questions, "word " * 100)
|
||||
# The state loses 50 tokens and the questions stay at the end.
|
||||
assert len(truncated) == strategy.max_length
|
||||
assert truncated[-200:] == prompt[-200:]
|
||||
assert truncated_schema == schema
|
||||
strategy.max_length = 100
|
||||
with pytest.raises(StructuredDecisionError, match="questions take"):
|
||||
strategy.encode(questions, "word")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_media_needs_a_model_that_takes_it(serving_and_engine):
|
||||
serving, engine = serving_and_engine
|
||||
request = StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST, images=["data:image/png;base64,AAAA"]
|
||||
)
|
||||
response = await serving.create_decision(request)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
assert "does not take image inputs" in response.error.message
|
||||
engine.generate.assert_not_called()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_reads_media_before_the_state(joint_schema, monkeypatch):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
strategy = serving.strategy
|
||||
strategy.__dict__["_connector"] = MediaConnector()
|
||||
image_pad = tokenizer.convert_tokens_to_ids("<|image_pad|>")
|
||||
rendered: dict[str, Any] = {}
|
||||
|
||||
async def process_for_engine_async(prompt, arrival_time):
|
||||
rendered.update(prompt)
|
||||
# The image's placeholder expands to three tokens.
|
||||
ids = [
|
||||
token
|
||||
for pad in prompt["prompt_token_ids"]
|
||||
for token in ([pad] * 3 if pad == image_pad else [pad])
|
||||
]
|
||||
return mm_input(ids, {}, {}, {}, cache_salt=prompt.get("cache_salt"))
|
||||
|
||||
renderer = strategy.context.online_renderer.renderer
|
||||
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
|
||||
buffer = BytesIO()
|
||||
Image.new("RGB", (8, 8), "red").save(buffer, format="PNG")
|
||||
image = "data:image/png;base64," + base64.b64encode(buffer.getvalue()).decode()
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(**JOINT_SCHEMA_REQUEST, images=[image])
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
|
||||
placeholder = tokenizer.encode(
|
||||
"<|vision_start|><|image_pad|><|vision_end|>\n", add_special_tokens=False
|
||||
)
|
||||
assert rendered["prompt_token_ids"] == strategy.prefix + placeholder
|
||||
assert rendered["multi_modal_data"]["image"][0].size == (8, 8)
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE}
|
||||
}
|
||||
prompt = engine.encode.call_args.args[0]
|
||||
assert prompt["type"] == "multimodal"
|
||||
ids = prompt["prompt_token_ids"]
|
||||
expanded = len(strategy.prefix) + len(placeholder) + 2
|
||||
assert ids[:expanded].count(image_pad) == 3
|
||||
state_and_questions = JOINT_SCHEMA_PROMPT.split("STATE:\n", 1)[1]
|
||||
assert tokenizer.decode(ids[expanded:]) == state_and_questions
|
||||
assert response.answers["team"]["choice"] == "billing"
|
||||
assert response.usage.input_tokens == len(ids)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_joint_schema_hears_video_soundtracks(joint_schema, monkeypatch):
|
||||
serving, engine, tokenizer = joint_schema
|
||||
strategy = serving.strategy
|
||||
video = MediaWithBytes(
|
||||
(np.zeros((4, 8, 8, 3), np.uint8), {"fps": 2.0}), b"video-bytes"
|
||||
)
|
||||
track = (np.zeros(160, np.float32), 16000)
|
||||
rendered: dict[str, Any] = {}
|
||||
|
||||
class FakeConnector:
|
||||
async def fetch_image_async(self, url):
|
||||
raise AssertionError("no images in this test")
|
||||
|
||||
async def load_from_url_async(self, url, media_io, *, fetch_timeout=None):
|
||||
return video
|
||||
|
||||
async def fetch_audio_async(self, url):
|
||||
if url == "https://example.org/silent.mp4":
|
||||
raise ValueError("No audio stream found.")
|
||||
return track
|
||||
|
||||
strategy.__dict__["_connector"] = FakeConnector()
|
||||
|
||||
async def process_for_engine_async(prompt, arrival_time):
|
||||
rendered.clear()
|
||||
rendered.update(prompt)
|
||||
return mm_input(prompt["prompt_token_ids"], {}, {}, {})
|
||||
|
||||
renderer = strategy.context.online_renderer.renderer
|
||||
monkeypatch.setattr(renderer, "process_for_engine_async", process_for_engine_async)
|
||||
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/talking.mp4"]
|
||||
)
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert rendered["multi_modal_data"] == {"video": [video], "audio": [track]}
|
||||
assert rendered["mm_processor_kwargs"] == {
|
||||
"images_kwargs": {"size": strategy.IMAGE_SIZE},
|
||||
"use_audio_in_video": True,
|
||||
}
|
||||
|
||||
# A video without a decodable soundtrack is watched silently.
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST, videos=["https://example.org/silent.mp4"]
|
||||
)
|
||||
)
|
||||
assert isinstance(response, StructuredDecisionResponse)
|
||||
assert rendered["multi_modal_data"] == {"video": [video]}
|
||||
assert rendered["mm_processor_kwargs"]["use_audio_in_video"] is False
|
||||
|
||||
# Audio clips cannot be read with videos that have soundtracks.
|
||||
response = await serving.create_decision(
|
||||
StructuredDecisionRequest(
|
||||
**JOINT_SCHEMA_REQUEST,
|
||||
videos=["https://example.org/talking.mp4"],
|
||||
audio=["https://example.org/clip.wav"],
|
||||
)
|
||||
)
|
||||
assert isinstance(response, ErrorResponse)
|
||||
assert response.error.code == 400
|
||||
|
||||
@@ -137,3 +137,32 @@ def test_loader_does_not_read_shards_the_model_skips(mtp_checkpoint, monkeypatch
|
||||
["model-00002-of-00003.safetensors", "model-00003-of-00003.safetensors"],
|
||||
sorted(f.name for f in mtp_checkpoint.glob("*.safetensors")),
|
||||
]
|
||||
|
||||
|
||||
def test_secondary_source_with_explicit_patterns_skips_backbone_index(tmp_path):
|
||||
"""A source with explicit allow patterns (e.g. a standalone head file in
|
||||
the repo root) must not be validated against the backbone's index."""
|
||||
save_file(
|
||||
{"backbone.weight": torch.zeros(1)},
|
||||
tmp_path / "model-00001-of-00001.safetensors",
|
||||
)
|
||||
save_file({"head.weight": torch.zeros(1)}, tmp_path / "joint_head.safetensors")
|
||||
(tmp_path / "model.safetensors.index.json").write_text(
|
||||
json.dumps(
|
||||
{"weight_map": {"backbone.weight": "model-00001-of-00001.safetensors"}}
|
||||
)
|
||||
)
|
||||
|
||||
loader = DefaultModelLoader(LoadConfig())
|
||||
secondary = DefaultModelLoader.Source(
|
||||
model_or_path=str(tmp_path),
|
||||
revision=None,
|
||||
prefix="pooler.head.",
|
||||
allow_patterns_overrides=["joint_head.safetensors"],
|
||||
)
|
||||
assert [n for n, _ in loader._get_weights_iterator(secondary)] == [
|
||||
"pooler.head.head.weight"
|
||||
]
|
||||
# The primary source still filters files the backbone index doesn't list.
|
||||
primary = DefaultModelLoader.Source(model_or_path=str(tmp_path), revision=None)
|
||||
assert [n for n, _ in loader._get_weights_iterator(primary)] == ["backbone.weight"]
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""The Clef-Omni pooler against a port of the checkpoint's joint schema head,
|
||||
which loops over questions where the pooler batches them."""
|
||||
|
||||
import math
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
|
||||
from vllm.model_executor.models.clef_omni import JointSchemaHead, JointSchemaPooler
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
|
||||
|
||||
HIDDEN, VOCAB = 64, 500
|
||||
TYPES = [0, 1, 1, 2]
|
||||
COUNTS = [2, 3, 1, 5]
|
||||
|
||||
|
||||
def reference_logits(head, hidden, token_ids, embedding, question_spans, spans):
|
||||
"""``JointSchemaHead.forward`` of the checkpoint's joint_schema_model.py
|
||||
for one record, with ``spans`` the option spans of each question."""
|
||||
hidden = head.hidden_norm(hidden)
|
||||
memory = head.memory_projection(hidden).unsqueeze(0)
|
||||
global_vector = hidden[-1]
|
||||
questions = torch.stack([hidden[s:e].mean(0) for s, e in question_spans])
|
||||
contexts = [torch.stack([hidden[s:e].mean(0) for s, e in q]) for q in spans]
|
||||
lexicals = [
|
||||
torch.stack([embedding[token_ids[s:e]].mean(0) for s, e in q]) for q in spans
|
||||
]
|
||||
options = torch.cat(
|
||||
[
|
||||
head.option_context_projection(c)
|
||||
+ head.option_lexical_projection(lex)
|
||||
+ head.option_question_projection(questions[i]).unsqueeze(0)
|
||||
for i, (c, lex) in enumerate(zip(contexts, lexicals))
|
||||
]
|
||||
).unsqueeze(0)
|
||||
for layer in head.evidence_layers:
|
||||
options = layer(options, memory)
|
||||
routed = torch.split(options[0], [len(q) for q in spans])
|
||||
|
||||
base = head.question_projection(questions)
|
||||
summaries = []
|
||||
for field, opts in zip(base, routed):
|
||||
weights = torch.softmax(opts @ field / math.sqrt(opts.shape[-1]), dim=0)
|
||||
summaries.append((weights[:, None] * opts).sum(0))
|
||||
fields = (
|
||||
base
|
||||
+ head.option_summary_norm(torch.stack(summaries))
|
||||
+ head.global_projection(global_vector)
|
||||
+ head.type_embedding(torch.tensor(TYPES))
|
||||
).unsqueeze(0)
|
||||
for layer in head.layers:
|
||||
fields = layer(fields, memory)
|
||||
fields = head.field_norm(fields[0])
|
||||
|
||||
logits = []
|
||||
for i, (field, lexical, opts) in enumerate(zip(fields, lexicals, routed)):
|
||||
anchor = F.normalize(questions[i] + global_vector, dim=-1)
|
||||
prior_scale = head.prior_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
prior = prior_scale * (F.normalize(lexical, dim=-1) @ anchor)
|
||||
opts = head.option_norm(opts)
|
||||
field = field.expand_as(opts)
|
||||
features = torch.cat([field, opts, field * opts, (field - opts).abs()], -1)
|
||||
joint_scale = head.joint_logit_scale.clamp(max=math.log(100.0)).exp()
|
||||
joint = joint_scale * F.cosine_similarity(field, opts, dim=-1)
|
||||
joint = joint + head.residual_scorer(features).squeeze(-1)
|
||||
logits.append(prior + torch.sigmoid(head.residual_gate) * joint)
|
||||
return logits
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def pooler():
|
||||
torch.manual_seed(0)
|
||||
head = JointSchemaHead(
|
||||
hidden_size=HIDDEN,
|
||||
width=32,
|
||||
routing_layers=2,
|
||||
layers=2,
|
||||
heads=4,
|
||||
feedforward=48,
|
||||
).eval()
|
||||
with torch.no_grad():
|
||||
for p in head.parameters():
|
||||
p.add_(torch.randn_like(p) * 0.2)
|
||||
embedding = torch.randn(VOCAB, HIDDEN)
|
||||
config = SimpleNamespace(
|
||||
scheduler_config=SimpleNamespace(
|
||||
enable_chunked_prefill=False, async_scheduling=False
|
||||
)
|
||||
)
|
||||
with patch(
|
||||
"vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
|
||||
return_value=config,
|
||||
):
|
||||
pooler = JointSchemaPooler(head, lambda ids: embedding[ids])
|
||||
return pooler, embedding
|
||||
|
||||
|
||||
def schema(length: int):
|
||||
"""Questions after a 20-token state, with options of 1 to 4 tokens."""
|
||||
pos = 20
|
||||
question_spans: list[tuple[int, int]] = []
|
||||
spans: list[list[tuple[int, int]]] = []
|
||||
for count in COUNTS:
|
||||
question_spans.append((pos, pos + 4))
|
||||
pos += 6
|
||||
spans.append([])
|
||||
for k in range(count):
|
||||
n = 1 + (k * 3) % 4
|
||||
spans[-1].append((pos, pos + n))
|
||||
pos += n + 1
|
||||
assert pos < length
|
||||
extra = {
|
||||
"question_types": TYPES,
|
||||
"question_spans": [[s - length, e - length] for s, e in question_spans],
|
||||
"option_spans": [[s - length, e - length] for q in spans for s, e in q],
|
||||
"option_counts": COUNTS,
|
||||
}
|
||||
return {"joint_schema": extra}, question_spans, spans
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_pooler_matches_reference(pooler):
|
||||
pooler, embedding = pooler
|
||||
lengths = [120, 90, 100]
|
||||
token_ids = [torch.randint(0, VOCAB, (n,)) for n in lengths]
|
||||
hidden = torch.randn(sum(lengths), HIDDEN)
|
||||
first, first_questions, first_spans = schema(lengths[0])
|
||||
last, last_questions, last_spans = schema(lengths[2])
|
||||
params = [
|
||||
PoolingParams(task="token_classify", use_activation=True, extra_kwargs=first),
|
||||
# A prompt without a schema, like the profiling run's.
|
||||
PoolingParams(task="token_classify", use_activation=True),
|
||||
PoolingParams(task="token_classify", use_activation=False, extra_kwargs=last),
|
||||
]
|
||||
padded = torch.zeros(len(lengths), max(lengths), dtype=torch.long)
|
||||
for i, ids in enumerate(token_ids):
|
||||
padded[i, : len(ids)] = ids
|
||||
metadata = PoolingMetadata(
|
||||
prompt_lens=torch.tensor(lengths),
|
||||
prompt_token_ids=None,
|
||||
prompt_token_ids_cpu=padded,
|
||||
pooling_params=params,
|
||||
pooling_states=[PoolingStates() for _ in lengths],
|
||||
)
|
||||
metadata.build_pooling_cursor(
|
||||
torch.tensor(lengths).numpy(),
|
||||
seq_lens_cpu=torch.tensor(lengths),
|
||||
device=torch.device("cpu"),
|
||||
)
|
||||
probs, empty, logits = pooler(hidden, metadata)
|
||||
|
||||
first_hidden, _, last_hidden = torch.split(hidden, lengths)
|
||||
expected = reference_logits(
|
||||
pooler.head, first_hidden, token_ids[0], embedding, first_questions, first_spans
|
||||
)
|
||||
expected_probs = torch.cat([q.softmax(-1) for q in expected])
|
||||
torch.testing.assert_close(probs, expected_probs, atol=1e-5, rtol=1e-4)
|
||||
assert empty.numel() == 0
|
||||
expected = reference_logits(
|
||||
pooler.head, last_hidden, token_ids[2], embedding, last_questions, last_spans
|
||||
)
|
||||
torch.testing.assert_close(logits, torch.cat(expected), atol=1e-5, rtol=1e-4)
|
||||
@@ -108,3 +108,33 @@ def test_longer_audio_generates_more_tokens(model_id: str) -> None:
|
||||
f"Expected longer audio (2s) to have more tokens than shorter (1s). "
|
||||
f"Got short={short_tokens}, long={long_tokens}"
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model_id", ["Cloudflare/clef-omni"])
|
||||
def test_audio_in_video_follows_requested_fps(model_id: str) -> None:
|
||||
"""Audio and video tokens interleave by the fps the video was processed
|
||||
at, as in HF, not by the default of one frame a second."""
|
||||
ctx = build_model_context(
|
||||
model_id,
|
||||
limit_mm_per_prompt={"audio": 1, "image": 0, "video": 1},
|
||||
)
|
||||
processor = MULTIMODAL_REGISTRY.create_processor(ctx.model_config)
|
||||
rng = np.random.RandomState(42)
|
||||
video = rng.randint(0, 255, (4, 64, 64, 3), dtype=np.uint8)
|
||||
audio = rng.rand(32000).astype(np.float32)
|
||||
prompt = "<|vision_start|><|video_pad|><|vision_end|>"
|
||||
mm_kwargs: dict[str, Any] = {
|
||||
"use_audio_in_video": True,
|
||||
"videos_kwargs": {"fps": 2.0, "do_sample_frames": False},
|
||||
}
|
||||
processed = processor(
|
||||
prompt,
|
||||
mm_items=processor.info.parse_mm_data(
|
||||
{"video": [video], "audio": [(audio, 16000)]}
|
||||
),
|
||||
hf_processor_mm_kwargs=mm_kwargs,
|
||||
)
|
||||
expected = processor.info.get_hf_processor()(
|
||||
text=[prompt], videos=[video], audio=[audio], return_tensors="pt", **mm_kwargs
|
||||
)
|
||||
assert processed["prompt_token_ids"] == expected["input_ids"][0].tolist()
|
||||
|
||||
@@ -1351,6 +1351,10 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"Qwen/Qwen3-Omni-30B-A3B-Instruct",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"ClefOmniForJointSchema": _HfExamplesInfo(
|
||||
"Cloudflare/clef-omni",
|
||||
max_model_len=4096,
|
||||
),
|
||||
"Qwen3ASRForConditionalGeneration": _HfExamplesInfo(
|
||||
"Qwen/Qwen3-ASR-0.6B",
|
||||
max_model_len=4096,
|
||||
|
||||
@@ -21,6 +21,7 @@ from vllm.models.minimax_m3.common.mm_preprocess import MiniMaxM3VideoBackend
|
||||
from vllm.multimodal.video import (
|
||||
PYNVVIDEOCODEC_VIDEO_BACKEND,
|
||||
VIDEO_LOADER_REGISTRY,
|
||||
ClefOmniVideoBackend,
|
||||
DynamicVideoBackend,
|
||||
Glm5NextVideoBackend,
|
||||
GLM46VVideoBackend,
|
||||
@@ -1813,3 +1814,18 @@ def test_glm5next_read_frames_dense_walk_matches_stock(tmp_path):
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
# One initial seek, then pure walking -- no re-seek churn.
|
||||
assert cap.seeks == 1
|
||||
|
||||
|
||||
def test_clef_omni_backend_samples_reference_frames(tmp_path):
|
||||
"""Clef-Omni samples the first frame at or after each 0.5 s boundary
|
||||
(the reference's 2 fps timestamp sampling), not uniform strides."""
|
||||
pytest.importorskip("cv2")
|
||||
|
||||
# 2 s at 30 fps: the reference reads frames 0, 15, 30 and 45, where
|
||||
# uniform sampling would read 0, 19, 39 and 59.
|
||||
path = _write_gray_video(tmp_path, total_frames=60, fps=30)
|
||||
frames, metadata = ClefOmniVideoBackend.load_bytes(path.read_bytes())
|
||||
assert metadata["frames_indices"] == [0, 15, 30, 45]
|
||||
assert metadata["do_sample_frames"] is False
|
||||
for frame, idx in zip(frames, [0, 15, 30, 45]):
|
||||
assert abs(round(float(np.asarray(frame).mean())) - idx) <= 1
|
||||
|
||||
Reference in New Issue
Block a user