diff --git a/docs/serving/online_serving/README.md b/docs/serving/online_serving/README.md index fc46a482c1..357ad0c5cb 100644 --- a/docs/serving/online_serving/README.md +++ b/docs/serving/online_serving/README.md @@ -16,6 +16,8 @@ We currently support the following OpenAI APIs: - [Chat Completions batch API](./openai_compatible_server.md#chat-api) (`/v1/chat/completions/batch`) - [Responses API](./openai_compatible_server.md#responses-api) (`/v1/responses`, `/v1/responses/{response_id}`, `/v1/responses/{response_id}/cancel`) - Only applicable to [text generation models](../../models/generative_models.md). +- [Decisions API](decisions.md) (`/v1/decisions`) + - Supports text-only predicate, choice, and score questions on supported generation models. - [Embeddings API](../../models/pooling_models/embed.md#openai-compatible-embeddings-api) (`/v1/embeddings`) - Only applicable to [embedding models](../../models/pooling_models/embed.md). - [Transcriptions API](./speech_to_text.md#transcriptions-api) (`/v1/audio/transcriptions`) diff --git a/docs/serving/online_serving/decisions.md b/docs/serving/online_serving/decisions.md new file mode 100644 index 0000000000..ed8446a357 --- /dev/null +++ b/docs/serving/online_serving/decisions.md @@ -0,0 +1,70 @@ +# Decisions API + +vLLM provides a text-only implementation of OpenAI's +[`POST /v1/decisions`](https://developers.openai.com/api/docs/guides/decisions) +API. Start a supported model: + +```bash +vllm serve Qwen/Qwen3-0.6B +``` + +```python +from openai import OpenAI + +client = OpenAI(base_url="http://localhost:8000/v1", api_key="EMPTY") +decision = client.decisions.create( + model="Qwen/Qwen3-0.6B", + input="The package arrived with a broken screen.", + questions=[ + { + "type": "predicate", + "name": "damaged", + "instructions": "Does the customer report a damaged item?", + }, + { + "type": "choice", + "name": "department", + "instructions": "Which department should handle this complaint?", + "choices": [{"value": "returns"}, {"value": "billing"}], + }, + { + "type": "score", + "instructions": "How severe is the damage?", + "levels": [{"label": "cosmetic"}, {"label": "broken"}], + }, + ], +) +print(decision.answers) +``` + +Use OpenAI Python SDK 3.26.0 or newer for `client.decisions.create`. +`input` accepts a string or an array of user messages with string content or +`input_text` parts. Answers retain question order and echo `name`, including +`null` for unnamed questions. Choice values preserve strings and booleans. +Scores are probability-weighted averages of zero-based level indices. + +The request contract follows the OpenAI OpenAPI schemas retrieved on +October 7, 2026. This MVP supports 1–64 questions, 2–26 choices, and 2–10 score +levels. Upstream permits 200 questions and 255 choices; this backend returns +a 400 when either model limit is exceeded. +The optional `safety_identifier` is caller metadata, available in request +body logging with `--enable-log-requests` and DEBUG logging; it does not +authenticate callers. + +## Implementation and limits + +This MVP uses the same label-reading backend as +[structured decisions](structured_decisions.md). It supports generation +architectures with a registered read strategy, including Qwen, DiffusionGemma, +and Liquid d1; unsupported models return 501. Each question reads a model's +label tokens. Usage reports the actual prompt and output tokens across these +reads, including prefix-cache counters. + +Probabilities are normalized over the supplied options. `confidence` is the +winning label's probability over the full vocabulary; it is not calibrated +to OpenAI's models. Thinking is disabled for these reads. + +Image inputs, other message roles, tools, streaming, and per-question refusal +scoring are outside this MVP. Unsupported request fields and input types are +rejected. The `/v1/systemone` endpoint retains its existing request format and +is available by default. diff --git a/docs/usage/security.md b/docs/usage/security.md index 428e64258e..f43ad0a6e3 100644 --- a/docs/usage/security.md +++ b/docs/usage/security.md @@ -175,6 +175,7 @@ When `--api-key` is configured, the following endpoints require Bearer token aut - `/v1/responses/render` - Render a self-contained response request (available on `vllm serve` only when `--enable-scale-out` is set, or on `vllm launch render` unless explicitly disabled) - `/v1/responses/{response_id}` - Retrieve a response - `/v1/responses/{response_id}/cancel` - Cancel a response +- `/v1/decisions` - Text decisions API - `/v1/score` - Scoring API - `/v1/rerank` - Reranking API - `/v1/systemone` - Structured decisions API diff --git a/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py b/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py index ea7c81ea26..ac812f0e11 100644 --- a/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py +++ b/tests/entrypoints/generate/generative_scoring/test_generative_scoring.py @@ -344,6 +344,35 @@ class TestGeneration: for item_result in result.data: assert 0.0 <= item_result.score <= 1.0 + @pytest.mark.asyncio + async def test_missing_read_returns_server_error_without_traceback( + self, monkeypatch + ): + mock_engine = _create_mock_engine() + serving = _create_serving(mock_engine) + + async def mock_generate(*args, **kwargs): + if False: + yield None + + mock_engine.generate = mock_generate + log_exception = MagicMock() + monkeypatch.setattr( + "vllm.entrypoints.generate.generative_scoring.serving.logger.exception", + log_exception, + ) + request = GenerativeScoringRequest( + model=MODEL_NAME, + query=[100], + items=[[200]], + label_token_ids=[1234, 5678], + ) + result = await serving.create_generative_scoring(request, None) + + assert isinstance(result, ErrorResponse) + assert result.error.code == 500 + log_exception.assert_not_called() + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/tests/entrypoints/generate/structured_decisions/test_decisions.py b/tests/entrypoints/generate/structured_decisions/test_decisions.py new file mode 100644 index 0000000000..f09396fb83 --- /dev/null +++ b/tests/entrypoints/generate/structured_decisions/test_decisions.py @@ -0,0 +1,240 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest +from pydantic import ValidationError + +from vllm.entrypoints.openai.decisions.adapters import ( + input_text, + make_answer, + make_read_question, +) +from vllm.entrypoints.openai.decisions.protocol import DecisionRequest +from vllm.entrypoints.openai.decisions.question_types import ( + StructuredDecisionError, +) + +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] + + +def request(**kwargs): + return DecisionRequest.model_validate( + { + "model": "test-model", + "input": "The package is broken.", + "questions": [{"type": "predicate", "instructions": "Is it damaged?"}], + **kwargs, + } + ) + + +def test_predicate_returns_probability_of_true_and_null_name(): + question = request().questions[0] + read = make_read_question(0, question) + assert [option.name for option in read.options] == ["false", "true"] + assert make_answer(question, [0.1, 0.9], 0.5).model_dump() == { + "type": "predicate", + "name": None, + "probability": 0.9, + } + + +def test_choice_preserves_boolean_and_string_values(): + question = request( + questions=[ + { + "type": "choice", + "name": "decision", + "instructions": "Select a value.", + "choices": [{"value": True}, {"value": "true"}, {"value": False}], + } + ] + ).questions[0] + read = make_read_question(0, question) + assert [option.name for option in read.options] == ["true", '"true"', "false"] + answer = make_answer(question, [0.7, 0.2, 0.1], 0.5).model_dump() + assert answer["name"] == "decision" + assert answer["choice"] is True + assert answer["probabilities"] == [ + {"value": True, "probability": 0.7}, + {"value": "true", "probability": 0.2}, + {"value": False, "probability": 0.1}, + ] + assert answer["confidence"] == pytest.approx(0.35) + + +def test_score_is_weighted_average_of_zero_based_level_indices(): + question = request( + questions=[ + { + "type": "score", + "instructions": "Rate severity.", + "levels": [{"label": label} for label in ("low", "medium", "high")], + } + ] + ).questions[0] + answer = make_answer(question, [0.1, 0.7, 0.2], 0.8).model_dump() + assert answer["score"] == pytest.approx(1.1) + assert answer["probabilities"] == [ + {"value": 0, "label": "low", "probability": 0.1}, + {"value": 1, "label": "medium", "probability": 0.7}, + {"value": 2, "label": "high", "probability": 0.2}, + ] + assert answer["confidence"] == pytest.approx(0.56) + + +def test_text_messages_preserve_evidence_order(): + parsed = request( + input=[ + {"role": "user", "content": "first"}, + { + "type": "message", + "role": "user", + "content": [ + {"type": "input_text", "text": "second"}, + {"type": "input_text", "text": "third"}, + ], + }, + ] + ) + assert input_text(parsed.input) == "first\n\nsecond\nthird" + assert input_text([]) == "" + + +@pytest.mark.parametrize("evidence", ["", [], [{"role": "user", "content": []}]]) +def test_empty_input_is_valid_under_the_openai_schema(evidence): + assert input_text(request(input=evidence).input) == "" + + +def test_score_levels_with_the_same_label_keep_their_positions(): + question = request( + questions=[ + { + "type": "score", + "instructions": "Rate severity.", + "levels": [ + {"label": "damage", "description": "cosmetic"}, + {"label": "damage", "description": "unusable"}, + ], + } + ] + ).questions[0] + read = make_read_question(0, question) + assert read.labels == ("A", "B") + answer = make_answer(question, [0.1, 0.9], 0.8).model_dump() + assert answer["score"] == pytest.approx(0.9) + assert [p["value"] for p in answer["probabilities"]] == [0, 1] + + +@pytest.mark.parametrize( + "override,match", + [ + ({"input": [{"role": "assistant", "content": "x"}]}, "user"), + ( + { + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": "data:image/png;base64,AA==", + } + ], + } + ] + }, + "text input only", + ), + ({"input": {"text": "x"}}, "Input should"), + ({"questions": []}, "at least 1"), + ( + {"questions": [{"type": "predicate", "instructions": "x", "name": None}]}, + "name must be a string", + ), + ({"questions": [{"type": "predicate", "instructions": 1}]}, "valid string"), + ( + {"questions": [{"type": "predicate", "instructions": "x", "criteria": {}}]}, + "Extra inputs", + ), + ({"safety_identifier": "x" * 129}, "at most 128"), + ({"stream": True}, "Extra inputs"), + ], +) +def test_rejects_invalid_or_unsupported_inputs(override, match): + with pytest.raises(ValidationError, match=match): + request(**override) + + +@pytest.mark.parametrize("values", [[True, True], ["a", "a"], [1, "a"], [None, "a"]]) +def test_choice_rejects_duplicates_and_non_string_non_boolean_values(values): + with pytest.raises(ValidationError): + request( + questions=[ + { + "type": "choice", + "instructions": "x", + "choices": [{"value": v} for v in values], + } + ] + ) + + +@pytest.mark.parametrize( + "count,valid", [(1, False), (2, True), (255, True), (256, False)] +) +def test_choice_limits(count, valid): + data = { + "questions": [ + { + "type": "choice", + "instructions": "x", + "choices": [{"value": str(i)} for i in range(count)], + } + ] + } + if valid: + assert len(request(**data).questions[0].choices) == count + else: + with pytest.raises(ValidationError): + request(**data) + + +@pytest.mark.parametrize( + "count,valid", [(1, False), (2, True), (10, True), (11, False)] +) +def test_score_limits(count, valid): + data = { + "questions": [ + { + "type": "score", + "instructions": "x", + "levels": [{"label": str(i)} for i in range(count)], + } + ] + } + if valid: + request(**data) + else: + with pytest.raises(ValidationError): + request(**data) + + +def test_question_limit_and_optional_safety_identifier(): + questions = [{"type": "predicate", "instructions": "x"}] * 200 + assert len(request(questions=questions, safety_identifier=None).questions) == 200 + with pytest.raises(ValidationError, match="at most 200"): + request(questions=[*questions, questions[0]]) + + +def test_backend_rejects_more_choices_than_single_token_labels(): + question = request( + questions=[ + { + "type": "choice", + "instructions": "x", + "choices": [{"value": str(i)} for i in range(27)], + } + ] + ).questions[0] + with pytest.raises(StructuredDecisionError, match="at most 26 choices"): + make_read_question(0, question) diff --git a/tests/entrypoints/generate/structured_decisions/test_question_types.py b/tests/entrypoints/generate/structured_decisions/test_question_types.py index 26047b5d6c..053ae99f27 100644 --- a/tests/entrypoints/generate/structured_decisions/test_question_types.py +++ b/tests/entrypoints/generate/structured_decisions/test_question_types.py @@ -4,7 +4,7 @@ from typing import Any import pytest -from vllm.entrypoints.generate.structured_decisions.question_types import ( +from vllm.entrypoints.openai.decisions.question_types import ( QUESTION_TYPES, Option, Question, diff --git a/tests/entrypoints/generate/structured_decisions/test_strategies.py b/tests/entrypoints/generate/structured_decisions/test_strategies.py index 505f0e35ee..8551726585 100644 --- a/tests/entrypoints/generate/structured_decisions/test_strategies.py +++ b/tests/entrypoints/generate/structured_decisions/test_strategies.py @@ -1,6 +1,10 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio import json +import logging +import math +from argparse import Namespace from io import BytesIO from types import SimpleNamespace from typing import Any @@ -13,6 +17,7 @@ import torch from fastapi import FastAPI, Request from fastapi.testclient import TestClient from PIL import Image +from starlette.datastructures import State from vllm.config import ModelConfig from vllm.engine.protocol import EngineClient @@ -23,16 +28,19 @@ from vllm.entrypoints.generate.structured_decisions.protocol import ( StructuredDecisionRequest, StructuredDecisionResponse, ) -from vllm.entrypoints.generate.structured_decisions.question_types import ( - LABELS, - StructuredDecisionError, - build_question, -) from vllm.entrypoints.generate.structured_decisions.serving import ( ServingStructuredDecisions, parse_questions, ) -from vllm.entrypoints.generate.structured_decisions.strategies import ( +from vllm.entrypoints.openai.decisions.api_router import register_decisions_api_router +from vllm.entrypoints.openai.decisions.question_types import ( + LABELS, + StructuredDecisionError, + build_question, +) +from vllm.entrypoints.openai.decisions.serving import OpenAIServingDecisions +from vllm.entrypoints.openai.decisions.state import init_decisions_state +from vllm.entrypoints.openai.decisions.strategies import ( ClefFlashStrategy, DiffusionGemmaCanvasStrategy, JointSchemaStrategy, @@ -50,7 +58,8 @@ from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.exception_handling.register import ( init_exception_handler, ) -from vllm.inputs import mm_input +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.inputs import mm_input, tokens_input from vllm.logprobs import Logprob from vllm.multimodal.media import MediaConnector, MediaWithBytes from vllm.outputs import ( @@ -67,6 +76,8 @@ TRACE_HEADERS = { "tracestate": "vendor=value", } +pytestmark = [pytest.mark.cpu_test, pytest.mark.skip_global_cleanup] + def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any: return SimpleNamespace(architecture=architecture, logprobs_mode=logprobs_mode) @@ -102,8 +113,10 @@ def test_strategy_selection(): def test_route_is_registered_by_default(): app = FastAPI() register_structured_decisions_api_router(app) + register_decisions_api_router(app) paths = {getattr(route, "path", None) for route in app.routes} assert "/v1/systemone" in paths + assert "/v1/decisions" in paths def test_unsupported_model_returns_501(): @@ -125,6 +138,18 @@ def test_unsupported_model_returns_501(): assert response.status_code == 501 +def test_decisions_route_returns_501_without_supported_strategy(): + app = FastAPI() + app.state.args = SimpleNamespace(log_error_stack=False) + app.state.openai_serving_decisions = None + init_exception_handler(app) + register_decisions_api_router(app) + + with TestClient(app) as client: + response = client.post("/v1/decisions", json=decision_body("decisions")) + assert response.status_code == 501 + + def test_structured_state_keeps_unicode_in_prompt(): state = {"message": "Français 日本語"} assert state_text(state) == '{"message": "Français 日本語"}' @@ -145,6 +170,44 @@ def qwen(): return tokenizer, prompt_ids +def test_decisions_state_initializes(qwen): + tokenizer, _ = qwen + engine = SimpleNamespace( + model_config=model("Qwen3ForCausalLM"), renderer=None, input_processor=None + ) + state = State() + state.online_renderer = SimpleNamespace( + renderer=SimpleNamespace(get_tokenizer=lambda: tokenizer) + ) + state.openai_serving_models = OpenAIServingModels( + engine, [BaseModelPath(name="test-model", model_path="test-model")] + ) + args = Namespace(chat_template_content_format="auto") + + strategy = init_decisions_state(engine, state, args, None, None, {}, ("generate",)) + + assert isinstance(strategy, NextTokenStrategy) + assert isinstance(state.openai_serving_decisions, OpenAIServingDecisions) + + +def test_unsupported_decisions_state_does_not_block_startup(): + engine = SimpleNamespace(model_config=model("Qwen3ForCausalLM", "raw_logits")) + state = State() + + strategy = init_decisions_state( + engine, + state, + Namespace(chat_template_content_format="auto"), + None, + None, + {}, + ("generate",), + ) + + assert strategy is None + assert state.openai_serving_decisions is None + + def test_labels_start_the_reply(qwen): tokenizer, prompt_ids = qwen tail, ids = reply_label_ids(tokenizer, prompt_ids) @@ -646,3 +709,165 @@ def test_liquid_caps_images_like_the_checkpoint(): small = Image.new("RGB", (640, 480)) assert cap_pixels(small, cap) is small assert NextTokenStrategy.max_image_pixels is None + + +@pytest.fixture +def decision_server(qwen): + tokenizer, _ = qwen + + class Engine: + model_config = SimpleNamespace(is_encoder_decoder=False) + renderer = input_processor = None + errored = False + failure = None + + def __init__(self): + self.admitted = [] + self.generated = [] + + def check_admission(self, count): + self.admitted.append(count) + + async def generate(self, prompt, params, request_id, **kwargs): + self.generated.append(prompt) + if self.failure == "bad_request": + raise ValueError("Invalid generation input") + if self.failure == "no_result": + return + ids = params.logprob_token_ids + logprobs = {i: Logprob(-math.log(len(ids))) for i in ids} + if self.failure == "missing_label": + logprobs.pop(ids[-1]) + output = CompletionOutput( + index=0, + text="A", + token_ids=[ids[0]], + cumulative_logprob=None, + logprobs=[logprobs], + finish_reason="error" if self.failure == "error" else "length", + ) + if self.failure == "no_logprobs": + output.logprobs = None + yield RequestOutput( + request_id=request_id, + prompt=None, + prompt_token_ids=prompt["prompt_token_ids"], + prompt_logprobs=None, + outputs=[] if self.failure == "no_output" else [output], + finished=True, + ) + + class Renderer: + renderer = SimpleNamespace(get_tokenizer=lambda: tokenizer) + + async def preprocess_chat(self, request, messages, **kwargs): + await asyncio.sleep(0) + ids = tokenizer.apply_chat_template( + messages, + add_generation_prompt=True, + enable_thinking=False, + tokenize=True, + return_dict=False, + ) + return [], [tokens_input(ids)] + + engine, renderer = Engine(), Renderer() + strategy = NextTokenStrategy(ReadContext(engine, renderer, None, "auto", {})) + models = OpenAIServingModels( + engine, [BaseModelPath(name="test-model", model_path="test-model")] + ) + request_logger = RequestLogger(max_log_len=None) + app = FastAPI() + app.state.args = Namespace(log_error_stack=False) + app.state.serving_structured_decisions = ServingStructuredDecisions( + models, strategy, request_logger=request_logger + ) + app.state.openai_serving_decisions = OpenAIServingDecisions( + models, strategy, request_logger=request_logger + ) + init_exception_handler(app) + register_structured_decisions_api_router(app) + register_decisions_api_router(app) + with TestClient(app, headers={"X-Request-Id": "test-request"}) as client: + yield SimpleNamespace( + client=client, engine=engine, renderer=renderer, tokenizer=tokenizer + ) + + +def decision_body(endpoint, count=1): + questions = [ + {"type": "choice", "instructions": f"Question {i}"} for i in range(count) + ] + if endpoint == "decisions": + return dict( + model="test-model", + input="evidence", + safety_identifier="test-user", + questions=[ + dict(q, choices=[{"value": "yes"}, {"value": "no"}]) for q in questions + ], + ) + return dict( + model="test-model", + state="evidence", + questions={ + str(i): dict(q, criteria={"yes": None, "no": None}) + for i, q in enumerate(questions) + }, + ) + + +@pytest.mark.parametrize("endpoint", ["decisions", "systemone"]) +@pytest.mark.parametrize("count,status", [(64, 200), (65, 400)]) +def test_model_question_limit_is_enforced_before_admission( + decision_server, endpoint, count, status +): + server = decision_server + response = server.client.post( + f"/v1/{endpoint}", json=decision_body(endpoint, count) + ) + assert response.status_code == status, response.text + assert server.engine.admitted == ([count] if status == 200 else []) + if status == 400: + assert "at most 64" in response.json()["error"]["message"] + assert not server.engine.generated + + +@pytest.mark.parametrize("endpoint", ["decisions", "systemone"]) +@pytest.mark.parametrize( + "failure", + ["no_result", "no_output", "error", "no_logprobs", "missing_label", "bad_request"], +) +def test_generation_failures_are_server_errors(decision_server, endpoint, failure): + server = decision_server + server.engine.failure = failure + response = server.client.post(f"/v1/{endpoint}", json=decision_body(endpoint)) + status = 400 if failure == "bad_request" else 500 + assert response.status_code == status, response.text + assert response.json()["error"]["type"] == ( + "BadRequestError" if status == 400 else "InternalServerError" + ) + assert len(server.engine.generated) == 1 + + +def test_decisions_logging_includes_receipt_and_body( + decision_server, caplog, monkeypatch +): + logger_name = "vllm.entrypoints.serve.utils.request_logger" + monkeypatch.setattr(logging.getLogger(logger_name), "propagate", True) + with caplog.at_level(logging.DEBUG, logger=logger_name): + response = decision_server.client.post( + "/v1/decisions", json=decision_body("decisions") + ) + assert response.status_code == 200, response.text + assert "Received request decision-test-request" in caplog.text + assert "evidence" in caplog.text + assert '"safety_identifier":"test-user"' in caplog.text + + +def test_prompts_retain_question_order(decision_server): + server = decision_server + response = server.client.post("/v1/decisions", json=decision_body("decisions", 2)) + assert response.status_code == 200, response.text + for i, prompt in enumerate(server.engine.generated): + assert f"Question {i}" in server.tokenizer.decode(prompt["prompt_token_ids"]) diff --git a/tests/entrypoints/openai/decisions/test_decisions_e2e.py b/tests/entrypoints/openai/decisions/test_decisions_e2e.py new file mode 100644 index 0000000000..ea079a2eaf --- /dev/null +++ b/tests/entrypoints/openai/decisions/test_decisions_e2e.py @@ -0,0 +1,101 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import pytest +import requests + +from tests.utils import RemoteOpenAIServer + +MODEL_NAME = "Qwen/Qwen3-0.6B" + + +@pytest.fixture(scope="module") +def server(): + args = [ + "--dtype", + "bfloat16", + "--max-model-len", + "1024", + "--enforce-eager", + "--max-num-seqs", + "32", + "--enable-prefix-caching", + ] + with RemoteOpenAIServer(MODEL_NAME, args) as remote_server: + yield remote_server + + +def test_openai_decisions_text_questions(server): + response = requests.post( + server.url_for("v1/decisions"), + json={ + "model": MODEL_NAME, + "input": "The package arrived with a broken screen.", + "questions": [ + {"type": "predicate", "instructions": "Is the screen broken?"}, + { + "type": "choice", + "name": "department", + "instructions": "Which department should handle this?", + "choices": [{"value": "returns"}, {"value": "billing"}], + }, + { + "type": "score", + "name": "damage", + "instructions": "Rate the damage.", + "levels": [{"label": "undamaged"}, {"label": "broken"}], + }, + ], + }, + ) + assert response.status_code == 200, response.text + body = response.json() + assert set(body) == {"model", "answers", "usage"} + predicate, choice_answer, score = body["answers"] + assert predicate["type"] == "predicate" + assert predicate["name"] is None + assert 0 <= predicate["probability"] <= 1 + assert choice_answer["type"] == "choice" + assert choice_answer["name"] == "department" + assert [p["value"] for p in choice_answer["probabilities"]] == [ + "returns", + "billing", + ] + assert ( + choice_answer["choice"] + == max(choice_answer["probabilities"], key=lambda p: p["probability"])["value"] + ) + assert sum( + p["probability"] for p in choice_answer["probabilities"] + ) == pytest.approx(1) + assert score["type"] == "score" + assert score["name"] == "damage" + assert [p["label"] for p in score["probabilities"]] == ["undamaged", "broken"] + assert score["score"] == pytest.approx( + sum(p["value"] * p["probability"] for p in score["probabilities"]) + ) + usage = body["usage"] + assert usage["output_tokens"] == 3 + assert usage["total_tokens"] == usage["input_tokens"] + usage["output_tokens"] + + +def test_openai_decisions_rejects_images(server): + response = requests.post( + server.url_for("v1/decisions"), + json={ + "model": MODEL_NAME, + "input": [ + { + "role": "user", + "content": [ + { + "type": "input_image", + "image_url": "https://example.com/image.png", + } + ], + } + ], + "questions": [{"type": "predicate", "instructions": "Is this red?"}], + }, + ) + assert response.status_code == 400 + assert "text input only" in response.json()["error"]["message"] diff --git a/tests/entrypoints/openai/test_openai_schema.py b/tests/entrypoints/openai/test_openai_schema.py index ae48fc1a9c..00b8b57aac 100644 --- a/tests/entrypoints/openai/test_openai_schema.py +++ b/tests/entrypoints/openai/test_openai_schema.py @@ -235,13 +235,16 @@ def test_openapi_stateless(case: schemathesis.Case): }.get(key, DEFAULT_TIMEOUT_SECONDS) # No need to verify SSL certificate for localhost - systemone = case.operation.path == "/v1/systemone" + unsupported_decisions = case.operation.path in { + "/v1/systemone", + "/v1/decisions", + } response = case.call_and_validate( verify=False, timeout=timeout, headers={"Content-Type": "application/json"}, - excluded_checks=[not_a_server_error] if systemone else None, + excluded_checks=[not_a_server_error] if unsupported_decisions else None, ) - if systemone: - # SmolVLM does not support structured decisions. + if unsupported_decisions: + # SmolVLM does not support either decision endpoint. assert response.status_code < 500 or response.status_code == 501 diff --git a/vllm/entrypoints/generate/generative_scoring/serving.py b/vllm/entrypoints/generate/generative_scoring/serving.py index 92c0847716..2e07a4d4ce 100644 --- a/vllm/entrypoints/generate/generative_scoring/serving.py +++ b/vllm/entrypoints/generate/generative_scoring/serving.py @@ -27,6 +27,7 @@ from vllm.entrypoints.serve.engine.protocol import ( ) from vllm.entrypoints.serve.engine.serving import BaseServing from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.exceptions import GenerationError from vllm.inputs import EngineInput, tokens_input from vllm.logger import init_logger from vllm.sampling_params import SamplingParams @@ -306,6 +307,8 @@ class ServingGenerativeScoring(BaseServing): return self.create_error_response("Client disconnected") except ValueError as e: return self.create_error_response(e) + except GenerationError as e: + return self.create_error_response(e) except Exception as e: logger.exception("Error during generation") return self.create_error_response(e) diff --git a/vllm/entrypoints/generate/label_reads.py b/vllm/entrypoints/generate/label_reads.py index eda61b22bc..c677ebc790 100644 --- a/vllm/entrypoints/generate/label_reads.py +++ b/vllm/entrypoints/generate/label_reads.py @@ -6,6 +6,7 @@ from collections.abc import AsyncGenerator, Mapping, Sequence from dataclasses import dataclass from vllm.engine.protocol import EngineClient +from vllm.exceptions import GenerationError from vllm.inputs import EngineInput from vllm.lora.request import LoRARequest from vllm.outputs import RequestOutput @@ -33,7 +34,7 @@ async def next_token_label_reads( ) -> list[LabelRead]: """Runs one read per input at once, as request ``{request_id}-{i}``. Each read's params set ``max_tokens=1`` and the label tokens as - ``logprob_token_ids``. Raises ValueError naming the item when a read has + ``logprob_token_ids``. Raises GenerationError naming the item when a read has no output or lacks a label's logprob.""" generators: list[AsyncGenerator[RequestOutput, None]] = [ engine_client.generate( @@ -53,14 +54,14 @@ async def next_token_label_reads( reads = [] for i, (result, params) in enumerate(zip(results, sampling_params)): if result is None: - raise ValueError(f"Failed to generate result for item {i}") + raise GenerationError(f"Failed to generate result for item {i}") if not result.outputs: - raise ValueError(f"No output generated for item {i}") + raise GenerationError(f"No output generated for item {i}") output = result.outputs[0] if output.finish_reason == "error": - raise ValueError(f"Generation error for item {i}") + raise GenerationError(f"Generation error for item {i}") if not output.logprobs: - raise ValueError( + raise GenerationError( f"No logprobs available for item {i}. " "This might indicate an issue with logprobs configuration." ) @@ -68,7 +69,7 @@ async def next_token_label_reads( label_ids = params.logprob_token_ids or [] missing = [t for t in label_ids if t not in logprobs] if missing: - raise ValueError( + raise GenerationError( f"Token IDs {missing} not found in logprobs for item {i}. " "This might indicate the tokens are outside the model's vocabulary." ) diff --git a/vllm/entrypoints/generate/structured_decisions/api_router.py b/vllm/entrypoints/generate/structured_decisions/api_router.py index a3afe7fe11..5dc8423bee 100644 --- a/vllm/entrypoints/generate/structured_decisions/api_router.py +++ b/vllm/entrypoints/generate/structured_decisions/api_router.py @@ -9,6 +9,7 @@ from fastapi.responses import JSONResponse from starlette.datastructures import State from vllm.engine.protocol import EngineClient +from vllm.entrypoints.openai.decisions.state import init_decisions_state from vllm.entrypoints.serve.engine.protocol import ErrorResponse from vllm.entrypoints.serve.utils.api_utils import ( load_aware_call, @@ -16,15 +17,11 @@ from vllm.entrypoints.serve.utils.api_utils import ( with_cancellation, ) from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.logger import init_logger from vllm.renderers.chat_utils import load_chat_template from vllm.tasks import SupportedTask from .protocol import StructuredDecisionRequest from .serving import ServingStructuredDecisions -from .strategies import ReadContext, select_read_strategy - -logger = init_logger(__name__) router = APIRouter() @@ -71,21 +68,19 @@ def init_structured_decisions_state( supported_tasks: tuple[SupportedTask, ...], default_chat_template_kwargs: dict[str, Any], ): - state.serving_structured_decisions = None - try: - strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks) - strategy = strategy_cls( - ReadContext( - engine_client=engine_client, - online_renderer=state.online_renderer, - chat_template=load_chat_template(args.chat_template), - chat_template_content_format=args.chat_template_content_format, - default_chat_template_kwargs=default_chat_template_kwargs, - ) + strategy = init_decisions_state( + engine_client, + state, + args, + request_logger, + load_chat_template(args.chat_template), + default_chat_template_kwargs, + supported_tasks, + ) + state.serving_structured_decisions = ( + ServingStructuredDecisions( + state.openai_serving_models, strategy, request_logger=request_logger ) - except ValueError as e: - logger.debug("/v1/systemone is disabled: %s", e) - return - state.serving_structured_decisions = ServingStructuredDecisions( - state.openai_serving_models, strategy, request_logger=request_logger + if strategy is not None + else None ) diff --git a/vllm/entrypoints/generate/structured_decisions/protocol.py b/vllm/entrypoints/generate/structured_decisions/protocol.py index a8f104b8cb..a5d4dc9d6a 100644 --- a/vllm/entrypoints/generate/structured_decisions/protocol.py +++ b/vllm/entrypoints/generate/structured_decisions/protocol.py @@ -7,11 +7,8 @@ from typing import Any, Literal from pydantic import Field, model_validator -from vllm.config import ModelConfig from vllm.entrypoints.generate.base.protocol import validate_cache_salt from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel -from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs -from vllm.renderers.chat_utils import ChatTemplateContentFormatOption from vllm.utils import random_uuid @@ -84,34 +81,3 @@ class StructuredDecisionResponse(OpenAIBaseModel): answers: dict[str, dict[str, Any]] usage: DecisionUsage diagnostics: dict[str, QuestionDiagnostics] - - -class ReadPromptRequest(OpenAIBaseModel): - """Chat options for one read's prompt: the state and the question in the - user turn, ending at the generation prompt so the label is the reply's - first token. Thinking is off unless the request turns it on, or the label - would follow a thought rather than start the reply.""" - - chat_template_kwargs: dict[str, Any] | None = None - cache_salt: str | None = None - - def build_chat_params( - self, - default_template: str | None, - default_template_content_format: ChatTemplateContentFormatOption, - ) -> ChatParams: - return ChatParams( - chat_template=default_template, - chat_template_content_format=default_template_content_format, - chat_template_kwargs=merge_kwargs( - merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs), - dict(add_generation_prompt=True, continue_final_message=False), - ), - ) - - def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: - return TokenizeParams( - max_total_tokens=model_config.max_model_len, - max_output_tokens=1, - add_special_tokens=False, - ) diff --git a/vllm/entrypoints/generate/structured_decisions/serving.py b/vllm/entrypoints/generate/structured_decisions/serving.py index 52fdd430b1..0af88db8f5 100644 --- a/vllm/entrypoints/generate/structured_decisions/serving.py +++ b/vllm/entrypoints/generate/structured_decisions/serving.py @@ -5,11 +5,14 @@ from typing import Any from fastapi import Request -from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.openai.decisions.question_types import ( + Question, + StructuredDecisionError, + build_question, +) +from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions +from vllm.entrypoints.openai.decisions.strategies import DecisionLimits from vllm.entrypoints.serve.engine.protocol import ErrorResponse -from vllm.entrypoints.serve.engine.serving import BaseServing -from vllm.entrypoints.serve.utils.request_logger import RequestLogger -from vllm.logger import init_logger from .protocol import ( DecisionUsage, @@ -17,10 +20,6 @@ from .protocol import ( StructuredDecisionRequest, StructuredDecisionResponse, ) -from .question_types import Question, StructuredDecisionError, build_question -from .strategies import DecisionLimits, ReadStrategy - -logger = init_logger(__name__) def parse_questions( @@ -51,22 +50,7 @@ def parse_questions( return questions -class ServingStructuredDecisions(BaseServing): - def __init__( - self, - models: OpenAIServingModels, - strategy: ReadStrategy, - *, - request_logger: RequestLogger | None = None, - ) -> None: - model_config = strategy.context.engine_client.model_config - super().__init__( - models=models, model_config=model_config, request_logger=request_logger - ) - self.engine_client = strategy.context.engine_client - self.strategy = strategy - self.limits = strategy.limits() - +class ServingStructuredDecisions(BaseServingDecisions): def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]: urls = { "image": request.images, diff --git a/vllm/entrypoints/launchers/api_server/routers.py b/vllm/entrypoints/launchers/api_server/routers.py index 52c995a301..273daee117 100644 --- a/vllm/entrypoints/launchers/api_server/routers.py +++ b/vllm/entrypoints/launchers/api_server/routers.py @@ -31,6 +31,12 @@ def register_api_routers( register_structured_decisions_api_router(app) + from vllm.entrypoints.openai.decisions.api_router import ( + register_decisions_api_router, + ) + + register_decisions_api_router(app) + from vllm.entrypoints.serve.sagemaker.api_router import ( attach_router as register_sagemaker_api_router, ) diff --git a/vllm/entrypoints/openai/decisions/__init__.py b/vllm/entrypoints/openai/decisions/__init__.py new file mode 100644 index 0000000000..208f01a7cb --- /dev/null +++ b/vllm/entrypoints/openai/decisions/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project diff --git a/vllm/entrypoints/openai/decisions/adapters.py b/vllm/entrypoints/openai/decisions/adapters.py new file mode 100644 index 0000000000..449ededb54 --- /dev/null +++ b/vllm/entrypoints/openai/decisions/adapters.py @@ -0,0 +1,91 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Translate Decisions questions and answers to and from label reads.""" + +import json + +from .protocol import ( + ChoiceAnswer, + ChoiceQuestion, + DecisionAnswer, + DecisionInputMessage, + DecisionQuestion, + PredicateAnswer, + PredicateQuestion, + ScoreAnswer, +) +from .question_types import ( + LABELS, + Option, + Question, + StructuredDecisionError, + argmax, +) +from .question_types import ( + ChoiceQuestion as LabelQuestion, +) + + +def make_read_question( + index: int, question: DecisionQuestion, max_options: int = len(LABELS) +) -> Question: + if isinstance(question, PredicateQuestion): + options = [Option("false"), Option("true")] + elif isinstance(question, ChoiceQuestion): + options = [ + Option(json.dumps(c.value, ensure_ascii=False), c.description) + for c in question.choices + ] + else: + options = [Option(level.label, level.description) for level in question.levels] + limit = min(max_options, len(LABELS)) + if len(options) > limit: + raise StructuredDecisionError( + f"This model supports at most {limit} choices per question" + ) + return Question( + id=str(index), + type=LabelQuestion(), + instructions=question.instructions, + options=tuple(options), + labels=LABELS[: len(options)], + ) + + +def make_answer( + question: DecisionQuestion, probs: list[float], label_mass: float +) -> DecisionAnswer: + if isinstance(question, PredicateQuestion): + return PredicateAnswer(name=question.name, probability=probs[1]) + top = argmax(probs) + confidence = probs[top] * label_mass + if isinstance(question, ChoiceQuestion): + return ChoiceAnswer( + name=question.name, + choice=question.choices[top].value, + probabilities=[ + {"value": choice.value, "probability": probability} + for choice, probability in zip(question.choices, probs) + ], + confidence=confidence, + ) + return ScoreAnswer( + name=question.name, + score=sum(i * probability for i, probability in enumerate(probs)), + probabilities=[ + {"value": i, "label": level.label, "probability": probability} + for i, (level, probability) in enumerate(zip(question.levels, probs)) + ], + confidence=confidence, + ) + + +def input_text(input: str | list[DecisionInputMessage]) -> str: + if isinstance(input, str): + return input + return "\n\n".join( + message.content + if isinstance(message.content, str) + else "\n".join(part.text for part in message.content) + for message in input + ) diff --git a/vllm/entrypoints/openai/decisions/api_router.py b/vllm/entrypoints/openai/decisions/api_router.py new file mode 100644 index 0000000000..dfcae2f0a4 --- /dev/null +++ b/vllm/entrypoints/openai/decisions/api_router.py @@ -0,0 +1,56 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from http import HTTPStatus + +from fastapi import APIRouter, Depends, FastAPI, Request +from fastapi.responses import JSONResponse + +from vllm.entrypoints.serve.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.exception_handling.error_response import ( + create_error_response, +) +from vllm.entrypoints.serve.utils.api_utils import ( + load_aware_call, + validate_json_request, + with_cancellation, +) + +from .protocol import DecisionRequest, DecisionResponse +from .serving import OpenAIServingDecisions + +router = APIRouter() + + +@router.post( + "/v1/decisions", + dependencies=[Depends(validate_json_request)], + response_model=DecisionResponse, + responses={ + status.value: {"model": ErrorResponse} + for status in ( + HTTPStatus.BAD_REQUEST, + HTTPStatus.NOT_FOUND, + HTTPStatus.INTERNAL_SERVER_ERROR, + HTTPStatus.NOT_IMPLEMENTED, + ) + }, +) +@with_cancellation +@load_aware_call +async def create_decision(request: DecisionRequest, raw_request: Request): + handler: OpenAIServingDecisions | None = getattr( + raw_request.app.state, "openai_serving_decisions", None + ) + if handler is None: + error = create_error_response( + NotImplementedError("The model does not support the Decisions API") + ) + return JSONResponse(content=error.model_dump(), status_code=error.error.code) + result = await handler.create_decisions(request, raw_request) + if isinstance(result, ErrorResponse): + return JSONResponse(content=result.model_dump(), status_code=result.error.code) + return JSONResponse(content=result.model_dump()) + + +def register_decisions_api_router(app: FastAPI): + app.include_router(router) diff --git a/vllm/entrypoints/openai/decisions/protocol.py b/vllm/entrypoints/openai/decisions/protocol.py new file mode 100644 index 0000000000..bdbb772dac --- /dev/null +++ b/vllm/entrypoints/openai/decisions/protocol.py @@ -0,0 +1,160 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""OpenAI Decisions API request and response models.""" + +from typing import Annotated, Any, Literal + +from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator + +from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel + +ShortText = Annotated[StrictStr, Field(max_length=1048576)] +InputText = Annotated[StrictStr, Field(max_length=10485760)] +ChoiceValue = StrictStr | StrictBool + + +class DecisionModel(OpenAIBaseModel): + model_config = ConfigDict(extra="forbid") + + +class DecisionInputText(DecisionModel): + type: Literal["input_text"] + text: InputText + + +class DecisionInputMessage(DecisionModel): + role: Literal["user"] + content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)] + type: Literal["message"] = "message" + + @model_validator(mode="before") + @classmethod + def reject_images(cls, data: Any) -> Any: + if ( + isinstance(data, dict) + and isinstance(data.get("content"), list) + and any( + isinstance(part, dict) and part.get("type") == "input_image" + for part in data["content"] + ) + ): + raise ValueError( + "Decisions supports text input only; images are not supported" + ) + return data + + +class QuestionBase(DecisionModel): + instructions: ShortText + name: ShortText | None = None + + @model_validator(mode="after") + def validate_name(self) -> "QuestionBase": + if "name" in self.model_fields_set and self.name is None: + raise ValueError("name must be a string when provided") + return self + + +class PredicateQuestion(QuestionBase): + type: Literal["predicate"] + + +class ChoiceOption(DecisionModel): + value: ChoiceValue + description: ShortText = "" + + +class ChoiceQuestion(QuestionBase): + type: Literal["choice"] + choices: list[ChoiceOption] = Field(min_length=2, max_length=255) + + @model_validator(mode="after") + def validate_choices(self) -> "ChoiceQuestion": + values = [(type(c.value), c.value) for c in self.choices] + if len(set(values)) != len(values): + raise ValueError("choice values must be distinct") + return self + + +class ScoreLevel(DecisionModel): + label: ShortText + description: ShortText = "" + + +class ScoreQuestion(QuestionBase): + type: Literal["score"] + levels: list[ScoreLevel] = Field(min_length=2, max_length=10) + + +DecisionQuestion = Annotated[ + PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type") +] + + +class DecisionRequest(DecisionModel): + model: ShortText + input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)] + questions: list[DecisionQuestion] = Field(min_length=1, max_length=200) + safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None + + +class PredicateAnswer(DecisionModel): + type: Literal["predicate"] = "predicate" + name: str | None + probability: float + + +class ChoiceProbability(DecisionModel): + value: ChoiceValue + probability: float + + +class ChoiceAnswer(DecisionModel): + type: Literal["choice"] = "choice" + name: str | None + choice: ChoiceValue + probabilities: list[ChoiceProbability] + confidence: float + + +class ScoreProbability(DecisionModel): + value: int + label: str + probability: float + + +class ScoreAnswer(DecisionModel): + type: Literal["score"] = "score" + name: str | None + score: float + probabilities: list[ScoreProbability] + confidence: float + + +DecisionAnswer = Annotated[ + PredicateAnswer | ChoiceAnswer | ScoreAnswer, + Field(discriminator="type"), +] + + +class InputTokensDetails(DecisionModel): + cached_tokens: int = 0 + cache_write_tokens: int = 0 + + +class OutputTokensDetails(DecisionModel): + reasoning_tokens: int = 0 + + +class OpenAIDecisionUsage(DecisionModel): + input_tokens: int + input_tokens_details: InputTokensDetails + output_tokens: int + output_tokens_details: OutputTokensDetails + total_tokens: int + + +class DecisionResponse(DecisionModel): + model: str + answers: list[DecisionAnswer] + usage: OpenAIDecisionUsage diff --git a/vllm/entrypoints/generate/structured_decisions/question_types.py b/vllm/entrypoints/openai/decisions/question_types.py similarity index 100% rename from vllm/entrypoints/generate/structured_decisions/question_types.py rename to vllm/entrypoints/openai/decisions/question_types.py diff --git a/vllm/entrypoints/openai/decisions/serving.py b/vllm/entrypoints/openai/decisions/serving.py new file mode 100644 index 0000000000..ac7b3253a0 --- /dev/null +++ b/vllm/entrypoints/openai/decisions/serving.py @@ -0,0 +1,95 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +import asyncio + +from fastapi import Request + +from vllm.entrypoints.openai.models.serving import OpenAIServingModels +from vllm.entrypoints.serve.engine.protocol import ErrorResponse +from vllm.entrypoints.serve.engine.serving import BaseServing +from vllm.entrypoints.serve.utils.request_logger import RequestLogger + +from .adapters import input_text, make_answer, make_read_question +from .protocol import ( + DecisionRequest, + DecisionResponse, + InputTokensDetails, + OpenAIDecisionUsage, + OutputTokensDetails, +) +from .question_types import StructuredDecisionError +from .strategies import ReadStrategy + + +class BaseServingDecisions(BaseServing): + def __init__( + self, + models: OpenAIServingModels, + strategy: ReadStrategy, + *, + request_logger: RequestLogger | None = None, + ) -> None: + super().__init__( + models=models, + model_config=strategy.context.engine_client.model_config, + request_logger=request_logger, + ) + self.strategy = strategy + self.limits = strategy.limits() + + +class OpenAIServingDecisions(BaseServingDecisions): + async def create_decisions( + self, request: DecisionRequest, raw_request: Request | None = None + ) -> DecisionResponse | ErrorResponse: + if (error := await self._check_model(request)) is not None: # type: ignore[arg-type] + return error + engine = self.strategy.context.engine_client + if engine.errored: + raise engine.dead_error + request_id = f"decision-{self._base_request_id(raw_request)}" + try: + if len(request.questions) > self.limits.max_questions: + raise StructuredDecisionError( + f"questions: at most {self.limits.max_questions} for this model" + ) + questions = [ + make_read_question(i, question, self.limits.max_options) + for i, question in enumerate(request.questions) + ] + lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type] + engine.check_admission(len(questions)) + text = input_text(request.input) + self._log_inputs(request_id, text, None, lora_request) + reads = await self.strategy.read( + questions, + None, + text, + request_id=request_id, + chat_template_kwargs=None, + lora_request=lora_request, + priority=0, + ) + except StructuredDecisionError as exc: + return self.create_error_response(exc) + except asyncio.CancelledError: + return self.create_error_response("Client disconnected") + input_tokens = sum(read.input_tokens for read in reads) + output_tokens = sum(read.output_tokens for read in reads) + return DecisionResponse( + model=self.models.model_name(lora_request), + answers=[ + make_answer(question, read.probs, read.label_mass) + for question, read in zip(request.questions, reads) + ], + usage=OpenAIDecisionUsage( + input_tokens=input_tokens, + output_tokens=output_tokens, + total_tokens=input_tokens + output_tokens, + input_tokens_details=InputTokensDetails( + cached_tokens=sum(read.cached_tokens for read in reads), + cache_write_tokens=sum(read.cache_write_tokens for read in reads), + ), + output_tokens_details=OutputTokensDetails(), + ), + ) diff --git a/vllm/entrypoints/openai/decisions/state.py b/vllm/entrypoints/openai/decisions/state.py new file mode 100644 index 0000000000..43e183a637 --- /dev/null +++ b/vllm/entrypoints/openai/decisions/state.py @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +from argparse import Namespace +from typing import Any + +from starlette.datastructures import State + +from vllm.engine.protocol import EngineClient +from vllm.entrypoints.serve.utils.request_logger import RequestLogger +from vllm.tasks import SupportedTask + +from .serving import OpenAIServingDecisions +from .strategies import ReadContext, ReadStrategy, select_read_strategy + + +def init_decisions_state( + engine_client: EngineClient, + state: State, + args: Namespace, + request_logger: RequestLogger | None, + chat_template: str | None, + default_chat_template_kwargs: dict[str, Any], + supported_tasks: tuple[SupportedTask, ...], +) -> ReadStrategy | None: + state.openai_serving_decisions = None + try: + strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks) + strategy = strategy_cls( + ReadContext( + engine_client=engine_client, + online_renderer=state.online_renderer, + chat_template=chat_template, + chat_template_content_format=args.chat_template_content_format, + default_chat_template_kwargs=default_chat_template_kwargs, + ) + ) + except ValueError: + return None + + if strategy.task == "generate": + state.openai_serving_decisions = OpenAIServingDecisions( + state.openai_serving_models, + strategy, + request_logger=request_logger, + ) + return strategy diff --git a/vllm/entrypoints/generate/structured_decisions/strategies.py b/vllm/entrypoints/openai/decisions/strategies.py similarity index 95% rename from vllm/entrypoints/generate/structured_decisions/strategies.py rename to vllm/entrypoints/openai/decisions/strategies.py index 66238cf7f1..30c7c65263 100644 --- a/vllm/entrypoints/generate/structured_decisions/strategies.py +++ b/vllm/entrypoints/openai/decisions/strategies.py @@ -24,6 +24,7 @@ import vllm.envs as envs from vllm.config import ModelConfig from vllm.engine.protocol import EngineClient from vllm.entrypoints.generate.label_reads import next_token_label_reads +from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel from vllm.inputs import EngineInput, MultiModalInput, TokensPrompt, tokens_input from vllm.logger import init_logger from vllm.lora.request import LoRARequest @@ -35,6 +36,7 @@ from vllm.multimodal.media import ( VideoMediaIO, ) from vllm.pooling_params import PoolingParams +from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs from vllm.renderers.chat_utils import ChatTemplateContentFormatOption from vllm.renderers.inputs.preprocess import extract_prompt_components from vllm.renderers.online_renderer import OnlineRenderer @@ -42,7 +44,6 @@ from vllm.sampling_params import SamplingParams from vllm.tasks import SupportedTask from vllm.tokenizers import TokenizerLike -from .protocol import ReadPromptRequest from .question_types import ( LABELS, QUESTION_TYPES, @@ -78,6 +79,37 @@ class QuestionRead: argmax_is_label: bool input_tokens: int output_tokens: int + cached_tokens: int = 0 + cache_write_tokens: int = 0 + + +class ReadPromptRequest(OpenAIBaseModel): + """Chat options for one read, ending at the generation prompt so the label + is the reply's first token. Thinking is off unless the request enables it.""" + + chat_template_kwargs: dict[str, Any] | None = None + cache_salt: str | None = None + + def build_chat_params( + self, + default_template: str | None, + default_template_content_format: ChatTemplateContentFormatOption, + ) -> ChatParams: + return ChatParams( + chat_template=default_template, + chat_template_content_format=default_template_content_format, + chat_template_kwargs=merge_kwargs( + merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs), + dict(add_generation_prompt=True, continue_final_message=False), + ), + ) + + def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams: + return TokenizeParams( + max_total_tokens=model_config.max_model_len, + max_output_tokens=1, + add_special_tokens=False, + ) def state_text(state: Any) -> str: @@ -350,6 +382,10 @@ class NextTokenStrategy(ReadStrategy): and any(output.token_ids[0] in ids for ids in groups), input_tokens=len(label_read.result.prompt_token_ids or ()), output_tokens=len(output.token_ids), + cached_tokens=label_read.result.num_cached_tokens or 0, + cache_write_tokens=( + label_read.result.num_cache_creation_tokens or 0 + ), ) ) return reads