[Feat] Add text-only OpenAI Decisions API MVP (#60465)
Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
Codex
Misha Goin
parent
ad8d39cbab
commit
c923e7769b
@@ -16,6 +16,8 @@ We currently support the following OpenAI APIs:
|
||||
- [Chat Completions batch API](./openai_compatible_server.md#chat-api) (`/v1/chat/completions/batch`)
|
||||
- [Responses API](./openai_compatible_server.md#responses-api) (`/v1/responses`, `/v1/responses/{response_id}`, `/v1/responses/{response_id}/cancel`)
|
||||
- Only applicable to [text generation models](../../models/generative_models.md).
|
||||
- [Decisions API](decisions.md) (`/v1/decisions`)
|
||||
- Supports text-only predicate, choice, and score questions on supported generation models.
|
||||
- [Embeddings API](../../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`)
|
||||
- Only applicable to [embedding models](../../models/pooling_models/embed.md).
|
||||
- [Transcriptions API](./speech_to_text.md#transcriptions-api) (`/v1/audio/transcriptions`)
|
||||
|
||||
@@ -0,0 +1,70 @@
|
||||
# Decisions API
|
||||
|
||||
vLLM provides a text-only implementation of OpenAI's
|
||||
[`POST /v1/decisions`](https://developers.openai.com/api/docs/guides/decisions)
|
||||
API. Start a supported model:
|
||||
|
||||
```bash
|
||||
vllm serve Qwen/Qwen3-0.6B
|
||||
```
|
||||
|
||||
```python
|
||||
from openai import OpenAI
|
||||
|
||||
client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY")
|
||||
decision = client.decisions.create(
|
||||
model="Qwen/Qwen3-0.6B",
|
||||
input="The package arrived with a broken screen.",
|
||||
questions=[
|
||||
{
|
||||
"type": "predicate",
|
||||
"name": "damaged",
|
||||
"instructions": "Does the customer report a damaged item?",
|
||||
},
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "department",
|
||||
"instructions": "Which department should handle this complaint?",
|
||||
"choices": [{"value": "returns"}, {"value": "billing"}],
|
||||
},
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "How severe is the damage?",
|
||||
"levels": [{"label": "cosmetic"}, {"label": "broken"}],
|
||||
},
|
||||
],
|
||||
)
|
||||
print(decision.answers)
|
||||
```
|
||||
|
||||
Use OpenAI Python SDK 3.26.0 or newer for `client.decisions.create`.
|
||||
`input` accepts a string or an array of user messages with string content or
|
||||
`input_text` parts. Answers retain question order and echo `name`, including
|
||||
`null` for unnamed questions. Choice values preserve strings and booleans.
|
||||
Scores are probability-weighted averages of zero-based level indices.
|
||||
|
||||
The request contract follows the OpenAI OpenAPI schemas retrieved on
|
||||
October 7, 2026. This MVP supports 1–64 questions, 2–26 choices, and 2–10 score
|
||||
levels. Upstream permits 200 questions and 255 choices; this backend returns
|
||||
a 400 when either model limit is exceeded.
|
||||
The optional `safety_identifier` is caller metadata, available in request
|
||||
body logging with `--enable-log-requests` and DEBUG logging; it does not
|
||||
authenticate callers.
|
||||
|
||||
## Implementation and limits
|
||||
|
||||
This MVP uses the same label-reading backend as
|
||||
[structured decisions](structured_decisions.md). It supports generation
|
||||
architectures with a registered read strategy, including Qwen, DiffusionGemma,
|
||||
and Liquid d1; unsupported models return 501. Each question reads a model's
|
||||
label tokens. Usage reports the actual prompt and output tokens across these
|
||||
reads, including prefix-cache counters.
|
||||
|
||||
Probabilities are normalized over the supplied options. `confidence` is the
|
||||
winning label's probability over the full vocabulary; it is not calibrated
|
||||
to OpenAI's models. Thinking is disabled for these reads.
|
||||
|
||||
Image inputs, other message roles, tools, streaming, and per-question refusal
|
||||
scoring are outside this MVP. Unsupported request fields and input types are
|
||||
rejected. The `/v1/systemone` endpoint retains its existing request format and
|
||||
is available by default.
|
||||
@@ -175,6 +175,7 @@ When `--api-key` is configured, the following endpoints require Bearer token aut
|
||||
- `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render` unless explicitly disabled)
|
||||
- `/v1/responses/{response_id}` - Retrieve a response
|
||||
- `/v1/responses/{response_id}/cancel` - Cancel a response
|
||||
- `/v1/decisions` - Text decisions API
|
||||
- `/v1/score` - Scoring API
|
||||
- `/v1/rerank` - Reranking API
|
||||
- `/v1/systemone` - Structured decisions API
|
||||
|
||||
@@ -344,6 +344,35 @@ class TestGeneration:
|
||||
for item_result in result.data:
|
||||
assert 0.0 <= item_result.score <= 1.0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_read_returns_server_error_without_traceback(
|
||||
self, monkeypatch
|
||||
):
|
||||
mock_engine = _create_mock_engine()
|
||||
serving = _create_serving(mock_engine)
|
||||
|
||||
async def mock_generate(*args, **kwargs):
|
||||
if False:
|
||||
yield None
|
||||
|
||||
mock_engine.generate = mock_generate
|
||||
log_exception = MagicMock()
|
||||
monkeypatch.setattr(
|
||||
"vllm.entrypoints.generate.generative_scoring.serving.logger.exception",
|
||||
log_exception,
|
||||
)
|
||||
request = GenerativeScoringRequest(
|
||||
model=MODEL_NAME,
|
||||
query=[100],
|
||||
items=[[200]],
|
||||
label_token_ids=[1234, 5678],
|
||||
)
|
||||
result = await serving.create_generative_scoring(request, None)
|
||||
|
||||
assert isinstance(result, ErrorResponse)
|
||||
assert result.error.code == 500
|
||||
log_exception.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.entrypoints.openai.decisions.adapters import (
|
||||
input_text,
|
||||
make_answer,
|
||||
make_read_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.protocol import DecisionRequest
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
StructuredDecisionError,
|
||||
)
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
def request(**kwargs):
|
||||
return DecisionRequest.model_validate(
|
||||
{
|
||||
"model": "test-model",
|
||||
"input": "The package is broken.",
|
||||
"questions": [{"type": "predicate", "instructions": "Is it damaged?"}],
|
||||
**kwargs,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def test_predicate_returns_probability_of_true_and_null_name():
|
||||
question = request().questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert [option.name for option in read.options] == ["false", "true"]
|
||||
assert make_answer(question, [0.1, 0.9], 0.5).model_dump() == {
|
||||
"type": "predicate",
|
||||
"name": None,
|
||||
"probability": 0.9,
|
||||
}
|
||||
|
||||
|
||||
def test_choice_preserves_boolean_and_string_values():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "decision",
|
||||
"instructions": "Select a value.",
|
||||
"choices": [{"value": True}, {"value": "true"}, {"value": False}],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert [option.name for option in read.options] == ["true", '"true"', "false"]
|
||||
answer = make_answer(question, [0.7, 0.2, 0.1], 0.5).model_dump()
|
||||
assert answer["name"] == "decision"
|
||||
assert answer["choice"] is True
|
||||
assert answer["probabilities"] == [
|
||||
{"value": True, "probability": 0.7},
|
||||
{"value": "true", "probability": 0.2},
|
||||
{"value": False, "probability": 0.1},
|
||||
]
|
||||
assert answer["confidence"] == pytest.approx(0.35)
|
||||
|
||||
|
||||
def test_score_is_weighted_average_of_zero_based_level_indices():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Rate severity.",
|
||||
"levels": [{"label": label} for label in ("low", "medium", "high")],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
answer = make_answer(question, [0.1, 0.7, 0.2], 0.8).model_dump()
|
||||
assert answer["score"] == pytest.approx(1.1)
|
||||
assert answer["probabilities"] == [
|
||||
{"value": 0, "label": "low", "probability": 0.1},
|
||||
{"value": 1, "label": "medium", "probability": 0.7},
|
||||
{"value": 2, "label": "high", "probability": 0.2},
|
||||
]
|
||||
assert answer["confidence"] == pytest.approx(0.56)
|
||||
|
||||
|
||||
def test_text_messages_preserve_evidence_order():
|
||||
parsed = request(
|
||||
input=[
|
||||
{"role": "user", "content": "first"},
|
||||
{
|
||||
"type": "message",
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "input_text", "text": "second"},
|
||||
{"type": "input_text", "text": "third"},
|
||||
],
|
||||
},
|
||||
]
|
||||
)
|
||||
assert input_text(parsed.input) == "first\n\nsecond\nthird"
|
||||
assert input_text([]) == ""
|
||||
|
||||
|
||||
@pytest.mark.parametrize("evidence", ["", [], [{"role": "user", "content": []}]])
|
||||
def test_empty_input_is_valid_under_the_openai_schema(evidence):
|
||||
assert input_text(request(input=evidence).input) == ""
|
||||
|
||||
|
||||
def test_score_levels_with_the_same_label_keep_their_positions():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "Rate severity.",
|
||||
"levels": [
|
||||
{"label": "damage", "description": "cosmetic"},
|
||||
{"label": "damage", "description": "unusable"},
|
||||
],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
read = make_read_question(0, question)
|
||||
assert read.labels == ("A", "B")
|
||||
answer = make_answer(question, [0.1, 0.9], 0.8).model_dump()
|
||||
assert answer["score"] == pytest.approx(0.9)
|
||||
assert [p["value"] for p in answer["probabilities"]] == [0, 1]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"override,match",
|
||||
[
|
||||
({"input": [{"role": "assistant", "content": "x"}]}, "user"),
|
||||
(
|
||||
{
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "data:image/png;base64,AA==",
|
||||
}
|
||||
],
|
||||
}
|
||||
]
|
||||
},
|
||||
"text input only",
|
||||
),
|
||||
({"input": {"text": "x"}}, "Input should"),
|
||||
({"questions": []}, "at least 1"),
|
||||
(
|
||||
{"questions": [{"type": "predicate", "instructions": "x", "name": None}]},
|
||||
"name must be a string",
|
||||
),
|
||||
({"questions": [{"type": "predicate", "instructions": 1}]}, "valid string"),
|
||||
(
|
||||
{"questions": [{"type": "predicate", "instructions": "x", "criteria": {}}]},
|
||||
"Extra inputs",
|
||||
),
|
||||
({"safety_identifier": "x" * 129}, "at most 128"),
|
||||
({"stream": True}, "Extra inputs"),
|
||||
],
|
||||
)
|
||||
def test_rejects_invalid_or_unsupported_inputs(override, match):
|
||||
with pytest.raises(ValidationError, match=match):
|
||||
request(**override)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("values", [[True, True], ["a", "a"], [1, "a"], [None, "a"]])
|
||||
def test_choice_rejects_duplicates_and_non_string_non_boolean_values(values):
|
||||
with pytest.raises(ValidationError):
|
||||
request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": v} for v in values],
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count,valid", [(1, False), (2, True), (255, True), (256, False)]
|
||||
)
|
||||
def test_choice_limits(count, valid):
|
||||
data = {
|
||||
"questions": [
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": str(i)} for i in range(count)],
|
||||
}
|
||||
]
|
||||
}
|
||||
if valid:
|
||||
assert len(request(**data).questions[0].choices) == count
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
request(**data)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"count,valid", [(1, False), (2, True), (10, True), (11, False)]
|
||||
)
|
||||
def test_score_limits(count, valid):
|
||||
data = {
|
||||
"questions": [
|
||||
{
|
||||
"type": "score",
|
||||
"instructions": "x",
|
||||
"levels": [{"label": str(i)} for i in range(count)],
|
||||
}
|
||||
]
|
||||
}
|
||||
if valid:
|
||||
request(**data)
|
||||
else:
|
||||
with pytest.raises(ValidationError):
|
||||
request(**data)
|
||||
|
||||
|
||||
def test_question_limit_and_optional_safety_identifier():
|
||||
questions = [{"type": "predicate", "instructions": "x"}] * 200
|
||||
assert len(request(questions=questions, safety_identifier=None).questions) == 200
|
||||
with pytest.raises(ValidationError, match="at most 200"):
|
||||
request(questions=[*questions, questions[0]])
|
||||
|
||||
|
||||
def test_backend_rejects_more_choices_than_single_token_labels():
|
||||
question = request(
|
||||
questions=[
|
||||
{
|
||||
"type": "choice",
|
||||
"instructions": "x",
|
||||
"choices": [{"value": str(i)} for i in range(27)],
|
||||
}
|
||||
]
|
||||
).questions[0]
|
||||
with pytest.raises(StructuredDecisionError, match="at most 26 choices"):
|
||||
make_read_question(0, question)
|
||||
@@ -4,7 +4,7 @@ from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.question_types import (
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
QUESTION_TYPES,
|
||||
Option,
|
||||
Question,
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
from argparse import Namespace
|
||||
from io import BytesIO
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
@@ -13,6 +17,7 @@ import torch
|
||||
from fastapi import FastAPI, Request
|
||||
from fastapi.testclient import TestClient
|
||||
from PIL import Image
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
@@ -23,16 +28,19 @@ from vllm.entrypoints.generate.structured_decisions.protocol import (
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.question_types import (
|
||||
LABELS,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.serving import (
|
||||
ServingStructuredDecisions,
|
||||
parse_questions,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
from vllm.entrypoints.openai.decisions.api_router import register_decisions_api_router
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
LABELS,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.serving import OpenAIServingDecisions
|
||||
from vllm.entrypoints.openai.decisions.state import init_decisions_state
|
||||
from vllm.entrypoints.openai.decisions.strategies import (
|
||||
ClefFlashStrategy,
|
||||
DiffusionGemmaCanvasStrategy,
|
||||
JointSchemaStrategy,
|
||||
@@ -50,7 +58,8 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.register import (
|
||||
init_exception_handler,
|
||||
)
|
||||
from vllm.inputs import mm_input
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import mm_input, tokens_input
|
||||
from vllm.logprobs import Logprob
|
||||
from vllm.multimodal.media import MediaConnector, MediaWithBytes
|
||||
from vllm.outputs import (
|
||||
@@ -67,6 +76,8 @@ TRACE_HEADERS = {
|
||||
"tracestate": "vendor=value",
|
||||
}
|
||||
|
||||
pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup]
|
||||
|
||||
|
||||
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
|
||||
return SimpleNamespace(architecture=architecture, logprobs_mode=logprobs_mode)
|
||||
@@ -102,8 +113,10 @@ def test_strategy_selection():
|
||||
def test_route_is_registered_by_default():
|
||||
app = FastAPI()
|
||||
register_structured_decisions_api_router(app)
|
||||
register_decisions_api_router(app)
|
||||
paths = {getattr(route, "path", None) for route in app.routes}
|
||||
assert "/v1/systemone" in paths
|
||||
assert "/v1/decisions" in paths
|
||||
|
||||
|
||||
def test_unsupported_model_returns_501():
|
||||
@@ -125,6 +138,18 @@ def test_unsupported_model_returns_501():
|
||||
assert response.status_code == 501
|
||||
|
||||
|
||||
def test_decisions_route_returns_501_without_supported_strategy():
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(log_error_stack=False)
|
||||
app.state.openai_serving_decisions = None
|
||||
init_exception_handler(app)
|
||||
register_decisions_api_router(app)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post("/v1/decisions", json=decision_body("decisions"))
|
||||
assert response.status_code == 501
|
||||
|
||||
|
||||
def test_structured_state_keeps_unicode_in_prompt():
|
||||
state = {"message": "Français 日本語"}
|
||||
assert state_text(state) == '{"message": "Français 日本語"}'
|
||||
@@ -145,6 +170,44 @@ def qwen():
|
||||
return tokenizer, prompt_ids
|
||||
|
||||
|
||||
def test_decisions_state_initializes(qwen):
|
||||
tokenizer, _ = qwen
|
||||
engine = SimpleNamespace(
|
||||
model_config=model("Qwen3ForCausalLM"), renderer=None, input_processor=None
|
||||
)
|
||||
state = State()
|
||||
state.online_renderer = SimpleNamespace(
|
||||
renderer=SimpleNamespace(get_tokenizer=lambda: tokenizer)
|
||||
)
|
||||
state.openai_serving_models = OpenAIServingModels(
|
||||
engine, [BaseModelPath(name="test-model", model_path="test-model")]
|
||||
)
|
||||
args = Namespace(chat_template_content_format="auto")
|
||||
|
||||
strategy = init_decisions_state(engine, state, args, None, None, {}, ("generate",))
|
||||
|
||||
assert isinstance(strategy, NextTokenStrategy)
|
||||
assert isinstance(state.openai_serving_decisions, OpenAIServingDecisions)
|
||||
|
||||
|
||||
def test_unsupported_decisions_state_does_not_block_startup():
|
||||
engine = SimpleNamespace(model_config=model("Qwen3ForCausalLM", "raw_logits"))
|
||||
state = State()
|
||||
|
||||
strategy = init_decisions_state(
|
||||
engine,
|
||||
state,
|
||||
Namespace(chat_template_content_format="auto"),
|
||||
None,
|
||||
None,
|
||||
{},
|
||||
("generate",),
|
||||
)
|
||||
|
||||
assert strategy is None
|
||||
assert state.openai_serving_decisions is None
|
||||
|
||||
|
||||
def test_labels_start_the_reply(qwen):
|
||||
tokenizer, prompt_ids = qwen
|
||||
tail, ids = reply_label_ids(tokenizer, prompt_ids)
|
||||
@@ -646,3 +709,165 @@ def test_liquid_caps_images_like_the_checkpoint():
|
||||
small = Image.new("RGB", (640, 480))
|
||||
assert cap_pixels(small, cap) is small
|
||||
assert NextTokenStrategy.max_image_pixels is None
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def decision_server(qwen):
|
||||
tokenizer, _ = qwen
|
||||
|
||||
class Engine:
|
||||
model_config = SimpleNamespace(is_encoder_decoder=False)
|
||||
renderer = input_processor = None
|
||||
errored = False
|
||||
failure = None
|
||||
|
||||
def __init__(self):
|
||||
self.admitted = []
|
||||
self.generated = []
|
||||
|
||||
def check_admission(self, count):
|
||||
self.admitted.append(count)
|
||||
|
||||
async def generate(self, prompt, params, request_id, **kwargs):
|
||||
self.generated.append(prompt)
|
||||
if self.failure == "bad_request":
|
||||
raise ValueError("Invalid generation input")
|
||||
if self.failure == "no_result":
|
||||
return
|
||||
ids = params.logprob_token_ids
|
||||
logprobs = {i: Logprob(-math.log(len(ids))) for i in ids}
|
||||
if self.failure == "missing_label":
|
||||
logprobs.pop(ids[-1])
|
||||
output = CompletionOutput(
|
||||
index=0,
|
||||
text="A",
|
||||
token_ids=[ids[0]],
|
||||
cumulative_logprob=None,
|
||||
logprobs=[logprobs],
|
||||
finish_reason="error" if self.failure == "error" else "length",
|
||||
)
|
||||
if self.failure == "no_logprobs":
|
||||
output.logprobs = None
|
||||
yield RequestOutput(
|
||||
request_id=request_id,
|
||||
prompt=None,
|
||||
prompt_token_ids=prompt["prompt_token_ids"],
|
||||
prompt_logprobs=None,
|
||||
outputs=[] if self.failure == "no_output" else [output],
|
||||
finished=True,
|
||||
)
|
||||
|
||||
class Renderer:
|
||||
renderer = SimpleNamespace(get_tokenizer=lambda: tokenizer)
|
||||
|
||||
async def preprocess_chat(self, request, messages, **kwargs):
|
||||
await asyncio.sleep(0)
|
||||
ids = tokenizer.apply_chat_template(
|
||||
messages,
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
tokenize=True,
|
||||
return_dict=False,
|
||||
)
|
||||
return [], [tokens_input(ids)]
|
||||
|
||||
engine, renderer = Engine(), Renderer()
|
||||
strategy = NextTokenStrategy(ReadContext(engine, renderer, None, "auto", {}))
|
||||
models = OpenAIServingModels(
|
||||
engine, [BaseModelPath(name="test-model", model_path="test-model")]
|
||||
)
|
||||
request_logger = RequestLogger(max_log_len=None)
|
||||
app = FastAPI()
|
||||
app.state.args = Namespace(log_error_stack=False)
|
||||
app.state.serving_structured_decisions = ServingStructuredDecisions(
|
||||
models, strategy, request_logger=request_logger
|
||||
)
|
||||
app.state.openai_serving_decisions = OpenAIServingDecisions(
|
||||
models, strategy, request_logger=request_logger
|
||||
)
|
||||
init_exception_handler(app)
|
||||
register_structured_decisions_api_router(app)
|
||||
register_decisions_api_router(app)
|
||||
with TestClient(app, headers={"X-Request-Id": "test-request"}) as client:
|
||||
yield SimpleNamespace(
|
||||
client=client, engine=engine, renderer=renderer, tokenizer=tokenizer
|
||||
)
|
||||
|
||||
|
||||
def decision_body(endpoint, count=1):
|
||||
questions = [
|
||||
{"type": "choice", "instructions": f"Question {i}"} for i in range(count)
|
||||
]
|
||||
if endpoint == "decisions":
|
||||
return dict(
|
||||
model="test-model",
|
||||
input="evidence",
|
||||
safety_identifier="test-user",
|
||||
questions=[
|
||||
dict(q, choices=[{"value": "yes"}, {"value": "no"}]) for q in questions
|
||||
],
|
||||
)
|
||||
return dict(
|
||||
model="test-model",
|
||||
state="evidence",
|
||||
questions={
|
||||
str(i): dict(q, criteria={"yes": None, "no": None})
|
||||
for i, q in enumerate(questions)
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["decisions", "systemone"])
|
||||
@pytest.mark.parametrize("count,status", [(64, 200), (65, 400)])
|
||||
def test_model_question_limit_is_enforced_before_admission(
|
||||
decision_server, endpoint, count, status
|
||||
):
|
||||
server = decision_server
|
||||
response = server.client.post(
|
||||
f"/v1/{endpoint}", json=decision_body(endpoint, count)
|
||||
)
|
||||
assert response.status_code == status, response.text
|
||||
assert server.engine.admitted == ([count] if status == 200 else [])
|
||||
if status == 400:
|
||||
assert "at most 64" in response.json()["error"]["message"]
|
||||
assert not server.engine.generated
|
||||
|
||||
|
||||
@pytest.mark.parametrize("endpoint", ["decisions", "systemone"])
|
||||
@pytest.mark.parametrize(
|
||||
"failure",
|
||||
["no_result", "no_output", "error", "no_logprobs", "missing_label", "bad_request"],
|
||||
)
|
||||
def test_generation_failures_are_server_errors(decision_server, endpoint, failure):
|
||||
server = decision_server
|
||||
server.engine.failure = failure
|
||||
response = server.client.post(f"/v1/{endpoint}", json=decision_body(endpoint))
|
||||
status = 400 if failure == "bad_request" else 500
|
||||
assert response.status_code == status, response.text
|
||||
assert response.json()["error"]["type"] == (
|
||||
"BadRequestError" if status == 400 else "InternalServerError"
|
||||
)
|
||||
assert len(server.engine.generated) == 1
|
||||
|
||||
|
||||
def test_decisions_logging_includes_receipt_and_body(
|
||||
decision_server, caplog, monkeypatch
|
||||
):
|
||||
logger_name = "vllm.entrypoints.serve.utils.request_logger"
|
||||
monkeypatch.setattr(logging.getLogger(logger_name), "propagate", True)
|
||||
with caplog.at_level(logging.DEBUG, logger=logger_name):
|
||||
response = decision_server.client.post(
|
||||
"/v1/decisions", json=decision_body("decisions")
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
assert "Received request decision-test-request" in caplog.text
|
||||
assert "evidence" in caplog.text
|
||||
assert '"safety_identifier":"test-user"' in caplog.text
|
||||
|
||||
|
||||
def test_prompts_retain_question_order(decision_server):
|
||||
server = decision_server
|
||||
response = server.client.post("/v1/decisions", json=decision_body("decisions", 2))
|
||||
assert response.status_code == 200, response.text
|
||||
for i, prompt in enumerate(server.engine.generated):
|
||||
assert f"Question {i}" in server.tokenizer.decode(prompt["prompt_token_ids"])
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enable-prefix-caching",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def test_openai_decisions_text_questions(server):
|
||||
response = requests.post(
|
||||
server.url_for("v1/decisions"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": "The package arrived with a broken screen.",
|
||||
"questions": [
|
||||
{"type": "predicate", "instructions": "Is the screen broken?"},
|
||||
{
|
||||
"type": "choice",
|
||||
"name": "department",
|
||||
"instructions": "Which department should handle this?",
|
||||
"choices": [{"value": "returns"}, {"value": "billing"}],
|
||||
},
|
||||
{
|
||||
"type": "score",
|
||||
"name": "damage",
|
||||
"instructions": "Rate the damage.",
|
||||
"levels": [{"label": "undamaged"}, {"label": "broken"}],
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert set(body) == {"model", "answers", "usage"}
|
||||
predicate, choice_answer, score = body["answers"]
|
||||
assert predicate["type"] == "predicate"
|
||||
assert predicate["name"] is None
|
||||
assert 0 <= predicate["probability"] <= 1
|
||||
assert choice_answer["type"] == "choice"
|
||||
assert choice_answer["name"] == "department"
|
||||
assert [p["value"] for p in choice_answer["probabilities"]] == [
|
||||
"returns",
|
||||
"billing",
|
||||
]
|
||||
assert (
|
||||
choice_answer["choice"]
|
||||
== max(choice_answer["probabilities"], key=lambda p: p["probability"])["value"]
|
||||
)
|
||||
assert sum(
|
||||
p["probability"] for p in choice_answer["probabilities"]
|
||||
) == pytest.approx(1)
|
||||
assert score["type"] == "score"
|
||||
assert score["name"] == "damage"
|
||||
assert [p["label"] for p in score["probabilities"]] == ["undamaged", "broken"]
|
||||
assert score["score"] == pytest.approx(
|
||||
sum(p["value"] * p["probability"] for p in score["probabilities"])
|
||||
)
|
||||
usage = body["usage"]
|
||||
assert usage["output_tokens"] == 3
|
||||
assert usage["total_tokens"] == usage["input_tokens"] + usage["output_tokens"]
|
||||
|
||||
|
||||
def test_openai_decisions_rejects_images(server):
|
||||
response = requests.post(
|
||||
server.url_for("v1/decisions"),
|
||||
json={
|
||||
"model": MODEL_NAME,
|
||||
"input": [
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_image",
|
||||
"image_url": "https://example.com/image.png",
|
||||
}
|
||||
],
|
||||
}
|
||||
],
|
||||
"questions": [{"type": "predicate", "instructions": "Is this red?"}],
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "text input only" in response.json()["error"]["message"]
|
||||
@@ -235,13 +235,16 @@ def test_openapi_stateless(case: schemathesis.Case):
|
||||
}.get(key, DEFAULT_TIMEOUT_SECONDS)
|
||||
|
||||
# No need to verify SSL certificate for localhost
|
||||
systemone = case.operation.path == "/v1/systemone"
|
||||
unsupported_decisions = case.operation.path in {
|
||||
"/v1/systemone",
|
||||
"/v1/decisions",
|
||||
}
|
||||
response = case.call_and_validate(
|
||||
verify=False,
|
||||
timeout=timeout,
|
||||
headers={"Content-Type": "application/json"},
|
||||
excluded_checks=[not_a_server_error] if systemone else None,
|
||||
excluded_checks=[not_a_server_error] if unsupported_decisions else None,
|
||||
)
|
||||
if systemone:
|
||||
# SmolVLM does not support structured decisions.
|
||||
if unsupported_decisions:
|
||||
# SmolVLM does not support either decision endpoint.
|
||||
assert response.status_code < 500 or response.status_code == 501
|
||||
|
||||
@@ -27,6 +27,7 @@ from vllm.entrypoints.serve.engine.protocol import (
|
||||
)
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.sampling_params import SamplingParams
|
||||
@@ -306,6 +307,8 @@ class ServingGenerativeScoring(BaseServing):
|
||||
return self.create_error_response("Client disconnected")
|
||||
except ValueError as e:
|
||||
return self.create_error_response(e)
|
||||
except GenerationError as e:
|
||||
return self.create_error_response(e)
|
||||
except Exception as e:
|
||||
logger.exception("Error during generation")
|
||||
return self.create_error_response(e)
|
||||
|
||||
@@ -6,6 +6,7 @@ from collections.abc import AsyncGenerator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.exceptions import GenerationError
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import RequestOutput
|
||||
@@ -33,7 +34,7 @@ async def next_token_label_reads(
|
||||
) -> list[LabelRead]:
|
||||
"""Runs one read per input at once, as request ``{request_id}-{i}``. Each
|
||||
read's params set ``max_tokens=1`` and the label tokens as
|
||||
``logprob_token_ids``. Raises ValueError naming the item when a read has
|
||||
``logprob_token_ids``. Raises GenerationError naming the item when a read has
|
||||
no output or lacks a label's logprob."""
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = [
|
||||
engine_client.generate(
|
||||
@@ -53,14 +54,14 @@ async def next_token_label_reads(
|
||||
reads = []
|
||||
for i, (result, params) in enumerate(zip(results, sampling_params)):
|
||||
if result is None:
|
||||
raise ValueError(f"Failed to generate result for item {i}")
|
||||
raise GenerationError(f"Failed to generate result for item {i}")
|
||||
if not result.outputs:
|
||||
raise ValueError(f"No output generated for item {i}")
|
||||
raise GenerationError(f"No output generated for item {i}")
|
||||
output = result.outputs[0]
|
||||
if output.finish_reason == "error":
|
||||
raise ValueError(f"Generation error for item {i}")
|
||||
raise GenerationError(f"Generation error for item {i}")
|
||||
if not output.logprobs:
|
||||
raise ValueError(
|
||||
raise GenerationError(
|
||||
f"No logprobs available for item {i}. "
|
||||
"This might indicate an issue with logprobs configuration."
|
||||
)
|
||||
@@ -68,7 +69,7 @@ async def next_token_label_reads(
|
||||
label_ids = params.logprob_token_ids or []
|
||||
missing = [t for t in label_ids if t not in logprobs]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
raise GenerationError(
|
||||
f"Token IDs {missing} not found in logprobs for item {i}. "
|
||||
"This might indicate the tokens are outside the model's vocabulary."
|
||||
)
|
||||
|
||||
@@ -9,6 +9,7 @@ from fastapi.responses import JSONResponse
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.decisions.state import init_decisions_state
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
@@ -16,15 +17,11 @@ from vllm.entrypoints.serve.utils.api_utils import (
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
from .strategies import ReadContext, select_read_strategy
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -71,21 +68,19 @@ def init_structured_decisions_state(
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
state.serving_structured_decisions = None
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=load_chat_template(args.chat_template),
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
strategy = init_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
load_chat_template(args.chat_template),
|
||||
default_chat_template_kwargs,
|
||||
supported_tasks,
|
||||
)
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models, strategy, request_logger=request_logger
|
||||
)
|
||||
except ValueError as e:
|
||||
logger.debug("/v1/systemone is disabled: %s", e)
|
||||
return
|
||||
state.serving_structured_decisions = ServingStructuredDecisions(
|
||||
state.openai_serving_models, strategy, request_logger=request_logger
|
||||
if strategy is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -7,11 +7,8 @@ from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
@@ -84,34 +81,3 @@ class StructuredDecisionResponse(OpenAIBaseModel):
|
||||
answers: dict[str, dict[str, Any]]
|
||||
usage: DecisionUsage
|
||||
diagnostics: dict[str, QuestionDiagnostics]
|
||||
|
||||
|
||||
class ReadPromptRequest(OpenAIBaseModel):
|
||||
"""Chat options for one read's prompt: the state and the question in the
|
||||
user turn, ending at the generation prompt so the label is the reply's
|
||||
first token. Thinking is off unless the request turns it on, or the label
|
||||
would follow a thought rather than start the reply."""
|
||||
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> ChatParams:
|
||||
return ChatParams(
|
||||
chat_template=default_template,
|
||||
chat_template_content_format=default_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs),
|
||||
dict(add_generation_prompt=True, continue_final_message=False),
|
||||
),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=1,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
|
||||
@@ -5,11 +5,14 @@ from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions
|
||||
from vllm.entrypoints.openai.decisions.strategies import DecisionLimits
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
@@ -17,10 +20,6 @@ from .protocol import (
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
from .question_types import Question, StructuredDecisionError, build_question
|
||||
from .strategies import DecisionLimits, ReadStrategy
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
@@ -51,22 +50,7 @@ def parse_questions(
|
||||
return questions
|
||||
|
||||
|
||||
class ServingStructuredDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
model_config = strategy.context.engine_client.model_config
|
||||
super().__init__(
|
||||
models=models, model_config=model_config, request_logger=request_logger
|
||||
)
|
||||
self.engine_client = strategy.context.engine_client
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
class ServingStructuredDecisions(BaseServingDecisions):
|
||||
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
|
||||
urls = {
|
||||
"image": request.images,
|
||||
|
||||
@@ -31,6 +31,12 @@ def register_api_routers(
|
||||
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.openai.decisions.api_router import (
|
||||
register_decisions_api_router,
|
||||
)
|
||||
|
||||
register_decisions_api_router(app)
|
||||
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
attach_router as register_sagemaker_api_router,
|
||||
)
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,91 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Translate Decisions questions and answers to and from label reads."""
|
||||
|
||||
import json
|
||||
|
||||
from .protocol import (
|
||||
ChoiceAnswer,
|
||||
ChoiceQuestion,
|
||||
DecisionAnswer,
|
||||
DecisionInputMessage,
|
||||
DecisionQuestion,
|
||||
PredicateAnswer,
|
||||
PredicateQuestion,
|
||||
ScoreAnswer,
|
||||
)
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
Option,
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
argmax,
|
||||
)
|
||||
from .question_types import (
|
||||
ChoiceQuestion as LabelQuestion,
|
||||
)
|
||||
|
||||
|
||||
def make_read_question(
|
||||
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
|
||||
) -> Question:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
options = [Option("false"), Option("true")]
|
||||
elif isinstance(question, ChoiceQuestion):
|
||||
options = [
|
||||
Option(json.dumps(c.value, ensure_ascii=False), c.description)
|
||||
for c in question.choices
|
||||
]
|
||||
else:
|
||||
options = [Option(level.label, level.description) for level in question.levels]
|
||||
limit = min(max_options, len(LABELS))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"This model supports at most {limit} choices per question"
|
||||
)
|
||||
return Question(
|
||||
id=str(index),
|
||||
type=LabelQuestion(),
|
||||
instructions=question.instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def make_answer(
|
||||
question: DecisionQuestion, probs: list[float], label_mass: float
|
||||
) -> DecisionAnswer:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
return PredicateAnswer(name=question.name, probability=probs[1])
|
||||
top = argmax(probs)
|
||||
confidence = probs[top] * label_mass
|
||||
if isinstance(question, ChoiceQuestion):
|
||||
return ChoiceAnswer(
|
||||
name=question.name,
|
||||
choice=question.choices[top].value,
|
||||
probabilities=[
|
||||
{"value": choice.value, "probability": probability}
|
||||
for choice, probability in zip(question.choices, probs)
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
return ScoreAnswer(
|
||||
name=question.name,
|
||||
score=sum(i * probability for i, probability in enumerate(probs)),
|
||||
probabilities=[
|
||||
{"value": i, "label": level.label, "probability": probability}
|
||||
for i, (level, probability) in enumerate(zip(question.levels, probs))
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def input_text(input: str | list[DecisionInputMessage]) -> str:
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
return "\n\n".join(
|
||||
message.content
|
||||
if isinstance(message.content, str)
|
||||
else "\n".join(part.text for part in message.content)
|
||||
for message in input
|
||||
)
|
||||
@@ -0,0 +1,56 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import DecisionRequest, DecisionResponse
|
||||
from .serving import OpenAIServingDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/decisions",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=DecisionResponse,
|
||||
responses={
|
||||
status.value: {"model": ErrorResponse}
|
||||
for status in (
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
HTTPStatus.NOT_FOUND,
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
HTTPStatus.NOT_IMPLEMENTED,
|
||||
)
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_decision(request: DecisionRequest, raw_request: Request):
|
||||
handler: OpenAIServingDecisions | None = getattr(
|
||||
raw_request.app.state, "openai_serving_decisions", None
|
||||
)
|
||||
if handler is None:
|
||||
error = create_error_response(
|
||||
NotImplementedError("The model does not support the Decisions API")
|
||||
)
|
||||
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
|
||||
result = await handler.create_decisions(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenAI Decisions API request and response models."""
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
|
||||
InputText = Annotated[StrictStr, Field(max_length=10485760)]
|
||||
ChoiceValue = StrictStr | StrictBool
|
||||
|
||||
|
||||
class DecisionModel(OpenAIBaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DecisionInputText(DecisionModel):
|
||||
type: Literal["input_text"]
|
||||
text: InputText
|
||||
|
||||
|
||||
class DecisionInputMessage(DecisionModel):
|
||||
role: Literal["user"]
|
||||
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
|
||||
type: Literal["message"] = "message"
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def reject_images(cls, data: Any) -> Any:
|
||||
if (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("content"), list)
|
||||
and any(
|
||||
isinstance(part, dict) and part.get("type") == "input_image"
|
||||
for part in data["content"]
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Decisions supports text input only; images are not supported"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class QuestionBase(DecisionModel):
|
||||
instructions: ShortText
|
||||
name: ShortText | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_name(self) -> "QuestionBase":
|
||||
if "name" in self.model_fields_set and self.name is None:
|
||||
raise ValueError("name must be a string when provided")
|
||||
return self
|
||||
|
||||
|
||||
class PredicateQuestion(QuestionBase):
|
||||
type: Literal["predicate"]
|
||||
|
||||
|
||||
class ChoiceOption(DecisionModel):
|
||||
value: ChoiceValue
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ChoiceQuestion(QuestionBase):
|
||||
type: Literal["choice"]
|
||||
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_choices(self) -> "ChoiceQuestion":
|
||||
values = [(type(c.value), c.value) for c in self.choices]
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError("choice values must be distinct")
|
||||
return self
|
||||
|
||||
|
||||
class ScoreLevel(DecisionModel):
|
||||
label: ShortText
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ScoreQuestion(QuestionBase):
|
||||
type: Literal["score"]
|
||||
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
|
||||
|
||||
|
||||
DecisionQuestion = Annotated[
|
||||
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
|
||||
]
|
||||
|
||||
|
||||
class DecisionRequest(DecisionModel):
|
||||
model: ShortText
|
||||
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
|
||||
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
|
||||
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
|
||||
|
||||
|
||||
class PredicateAnswer(DecisionModel):
|
||||
type: Literal["predicate"] = "predicate"
|
||||
name: str | None
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceProbability(DecisionModel):
|
||||
value: ChoiceValue
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceAnswer(DecisionModel):
|
||||
type: Literal["choice"] = "choice"
|
||||
name: str | None
|
||||
choice: ChoiceValue
|
||||
probabilities: list[ChoiceProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
class ScoreProbability(DecisionModel):
|
||||
value: int
|
||||
label: str
|
||||
probability: float
|
||||
|
||||
|
||||
class ScoreAnswer(DecisionModel):
|
||||
type: Literal["score"] = "score"
|
||||
name: str | None
|
||||
score: float
|
||||
probabilities: list[ScoreProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
DecisionAnswer = Annotated[
|
||||
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class InputTokensDetails(DecisionModel):
|
||||
cached_tokens: int = 0
|
||||
cache_write_tokens: int = 0
|
||||
|
||||
|
||||
class OutputTokensDetails(DecisionModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class OpenAIDecisionUsage(DecisionModel):
|
||||
input_tokens: int
|
||||
input_tokens_details: InputTokensDetails
|
||||
output_tokens: int
|
||||
output_tokens_details: OutputTokensDetails
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class DecisionResponse(DecisionModel):
|
||||
model: str
|
||||
answers: list[DecisionAnswer]
|
||||
usage: OpenAIDecisionUsage
|
||||
@@ -0,0 +1,95 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
from .adapters import input_text, make_answer, make_read_question
|
||||
from .protocol import (
|
||||
DecisionRequest,
|
||||
DecisionResponse,
|
||||
InputTokensDetails,
|
||||
OpenAIDecisionUsage,
|
||||
OutputTokensDetails,
|
||||
)
|
||||
from .question_types import StructuredDecisionError
|
||||
from .strategies import ReadStrategy
|
||||
|
||||
|
||||
class BaseServingDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=strategy.context.engine_client.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
|
||||
class OpenAIServingDecisions(BaseServingDecisions):
|
||||
async def create_decisions(
|
||||
self, request: DecisionRequest, raw_request: Request | None = None
|
||||
) -> DecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine = self.strategy.context.engine_client
|
||||
if engine.errored:
|
||||
raise engine.dead_error
|
||||
request_id = f"decision-{self._base_request_id(raw_request)}"
|
||||
try:
|
||||
if len(request.questions) > self.limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {self.limits.max_questions} for this model"
|
||||
)
|
||||
questions = [
|
||||
make_read_question(i, question, self.limits.max_options)
|
||||
for i, question in enumerate(request.questions)
|
||||
]
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine.check_admission(len(questions))
|
||||
text = input_text(request.input)
|
||||
self._log_inputs(request_id, text, None, lora_request)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
None,
|
||||
text,
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=None,
|
||||
lora_request=lora_request,
|
||||
priority=0,
|
||||
)
|
||||
except StructuredDecisionError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
input_tokens = sum(read.input_tokens for read in reads)
|
||||
output_tokens = sum(read.output_tokens for read in reads)
|
||||
return DecisionResponse(
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=[
|
||||
make_answer(question, read.probs, read.label_mass)
|
||||
for question, read in zip(request.questions, reads)
|
||||
],
|
||||
usage=OpenAIDecisionUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=input_tokens + output_tokens,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cached_tokens=sum(read.cached_tokens for read in reads),
|
||||
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(),
|
||||
),
|
||||
)
|
||||
@@ -0,0 +1,46 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .serving import OpenAIServingDecisions
|
||||
from .strategies import ReadContext, ReadStrategy, select_read_strategy
|
||||
|
||||
|
||||
def init_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
) -> ReadStrategy | None:
|
||||
state.openai_serving_decisions = None
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if strategy.task == "generate":
|
||||
state.openai_serving_decisions = OpenAIServingDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
return strategy
|
||||
+37
-1
@@ -24,6 +24,7 @@ import vllm.envs as envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.label_reads import next_token_label_reads
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.request import LoRARequest
|
||||
@@ -35,6 +36,7 @@ from vllm.multimodal.media import (
|
||||
VideoMediaIO,
|
||||
)
|
||||
from vllm.pooling_params import PoolingParams
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.renderers.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
@@ -42,7 +44,6 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tasks import SupportedTask
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .protocol import ReadPromptRequest
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
QUESTION_TYPES,
|
||||
@@ -78,6 +79,37 @@ class QuestionRead:
|
||||
argmax_is_label: bool
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
cached_tokens: int = 0
|
||||
cache_write_tokens: int = 0
|
||||
|
||||
|
||||
class ReadPromptRequest(OpenAIBaseModel):
|
||||
"""Chat options for one read, ending at the generation prompt so the label
|
||||
is the reply's first token. Thinking is off unless the request enables it."""
|
||||
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> ChatParams:
|
||||
return ChatParams(
|
||||
chat_template=default_template,
|
||||
chat_template_content_format=default_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs),
|
||||
dict(add_generation_prompt=True, continue_final_message=False),
|
||||
),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=1,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
|
||||
|
||||
def state_text(state: Any) -> str:
|
||||
@@ -350,6 +382,10 @@ class NextTokenStrategy(ReadStrategy):
|
||||
and any(output.token_ids[0] in ids for ids in groups),
|
||||
input_tokens=len(label_read.result.prompt_token_ids or ()),
|
||||
output_tokens=len(output.token_ids),
|
||||
cached_tokens=label_read.result.num_cached_tokens or 0,
|
||||
cache_write_tokens=(
|
||||
label_read.result.num_cache_creation_tokens or 0
|
||||
),
|
||||
)
|
||||
)
|
||||
return reads
|
||||
Reference in New Issue
Block a user