[Feat] Structured decisions endpoint (/v1/systemone) (#59299)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Matt Mastracci
2026-10-08 14:48:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Lucas Wilkinson Lucas Wilkinson Francisco Javier Arceo Chauncey
parent 273e099693
commit d33872dfeb
17 changed files with 1285 additions and 61 deletions
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import StructuredDecisionRequest
from .serving import ServingStructuredDecisions
router = APIRouter()
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
return getattr(request.app.state, "serving_structured_decisions", None)
@router.post(
"/v1/systemone",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_structured_decision(
request: StructuredDecisionRequest, raw_request: Request
):
handler = structured_decisions(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support the structured decision API"
)
result = await handler.create_decision(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_structured_decisions_api_router(app: FastAPI):
if getattr(app.state.args, "enable_structured_decisions", False):
app.include_router(router)
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Request and response models of the structured decisions API (/v1/systemone)."""
import time
from typing import Any, Literal
from pydantic import Field
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
from vllm.utils import random_uuid
class QuestionSpec(OpenAIBaseModel):
type: str
instructions: Any = ""
criteria: Any = None
class StructuredDecisionRequest(OpenAIBaseModel):
model: str | None = None
state: Any = Field(..., description="What the questions are about: text or JSON.")
questions: dict[str, QuestionSpec] = Field(
..., description="Question id to question, asked in this order."
)
instructions: str | None = Field(
default=None, description="Context placed ahead of the questions."
)
chat_template_kwargs: dict[str, Any] | None = None
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
request_id: str = Field(default_factory=random_uuid)
class DecisionUsage(OpenAIBaseModel):
input_tokens: int
output_tokens: int
class QuestionDiagnostics(OpenAIBaseModel):
label_mass: float = Field(
description="Probability the model put on the labels, over the full "
"vocabulary. A low value means most of the probability went to tokens "
"that are not labels."
)
argmax_is_label: bool
class StructuredDecisionResponse(OpenAIBaseModel):
id: str
object: Literal["structured_decision"] = "structured_decision"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
answers: dict[str, dict[str, Any]]
usage: DecisionUsage
diagnostics: dict[str, QuestionDiagnostics]
class ReadPromptRequest(OpenAIBaseModel):
"""Chat options for one read's prompt: the state and the question in the
user turn, ending at the generation prompt so the label is the reply's
first token. Thinking is off unless the request turns it on, or the label
would follow a thought rather than start the reply."""
chat_template_kwargs: dict[str, Any] | None = None
def build_chat_params(
self,
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
) -> ChatParams:
return ChatParams(
chat_template=default_template,
chat_template_content_format=default_template_content_format,
chat_template_kwargs=merge_kwargs(
merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs),
dict(add_generation_prompt=True, continue_final_message=False),
),
)
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return TokenizeParams(
max_total_tokens=model_config.max_model_len,
max_output_tokens=1,
add_special_tokens=False,
)
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Question types for structured decisions.
A question type turns a request's criteria into options, writes the question
for the model with a label per option, and builds the answer from the label
probabilities.
"""
import math
import string
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: Options are labeled A to Z in order. A single capital letter is one token
#: at the start of a reply for the tokenizers tested; every read checks it.
LABELS = tuple(string.ascii_uppercase)
@dataclass(frozen=True)
class Option:
name: str
description: str | None = None
@dataclass(frozen=True)
class Question:
id: str
type: "QuestionType"
instructions: str
options: tuple[Option, ...]
labels: tuple[str, ...]
class QuestionType(ABC):
name: ClassVar[str]
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@abstractmethod
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
"""The answer for ``question``. ``probs[i]`` is the probability of
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(
f"{label}: {o.name} - {o.description}"
if o.description
else f"{label}: {o.name}"
)
lines.append("Answer with the letter of one option only.")
return "\n".join(lines)
QUESTION_TYPES: dict[str, QuestionType] = {}
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
if cls.name in QUESTION_TYPES:
raise ValueError(f"question type {cls.name!r} is already registered")
QUESTION_TYPES[cls.name] = cls()
return cls
def get_question_type(name: str) -> QuestionType:
try:
return QUESTION_TYPES[name]
except KeyError:
raise StructuredDecisionError(
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
) from None
def build_question(
qid: str,
type_name: str,
instructions: Any,
criteria: Any,
max_options: int,
) -> Question:
if not qid:
raise StructuredDecisionError("question ids must be non-empty")
qtype = get_question_type(type_name)
options = qtype.parse_options(qid, criteria)
if not options:
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(LABELS))
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
)
if not isinstance(instructions, str):
instructions = "" if instructions is None else str(instructions)
return Question(
id=qid,
type=qtype,
instructions=instructions,
options=tuple(options),
labels=LABELS[: len(options)],
)
def argmax(values: list[float]) -> int:
return max(range(len(values)), key=values.__getitem__)
def label_softmax(logprobs: list[float]) -> list[float]:
"""Softmax of the label logprobs."""
top = max(logprobs)
weights = [math.exp(lp - top) for lp in logprobs]
total = sum(weights)
return [w / total for w in weights]
@register_question_type
class ChoiceQuestion(QuestionType):
"""Pick one option. ``criteria`` maps each option name to a description,
or to null."""
name = "choice"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, dict) or not criteria:
raise StructuredDecisionError(
f"question {qid!r}: choice criteria must map option names to "
"a description or null"
)
return [
Option(str(name), None if desc is None else str(desc))
for name, desc in criteria.items()
]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
"choice": question.options[top].name,
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
# The chosen label's probability over the whole vocabulary, so a
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
from typing import Any
from fastapi import Request
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from .protocol import (
DecisionUsage,
QuestionDiagnostics,
StructuredDecisionRequest,
StructuredDecisionResponse,
)
from .question_types import Question, StructuredDecisionError, build_question
from .strategies import DecisionLimits, ReadStrategy
logger = init_logger(__name__)
def state_text(state: Any) -> str:
return state if isinstance(state, str) else json.dumps(state)
def parse_questions(
request: StructuredDecisionRequest,
limits: DecisionLimits,
) -> list[Question]:
if not request.questions:
raise StructuredDecisionError("questions: needs at least one question")
if len(request.questions) > limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {limits.max_questions} for this model"
)
questions = []
for qid, spec in request.questions.items():
if spec.model_extra:
raise StructuredDecisionError(
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
)
questions.append(
build_question(
qid,
spec.type,
spec.instructions,
spec.criteria,
limits.max_options,
)
)
return questions
class ServingStructuredDecisions(BaseServing):
def __init__(
self,
models: OpenAIServingModels,
strategy: ReadStrategy,
*,
request_logger: RequestLogger | None = None,
) -> None:
model_config = strategy.context.engine_client.model_config
super().__init__(
models=models, model_config=model_config, request_logger=request_logger
)
self.strategy = strategy
self.limits = strategy.limits()
async def create_decision(
self,
request: StructuredDecisionRequest,
raw_request: Request | None = None,
) -> StructuredDecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine_client = self.strategy.context.engine_client
if engine_client.errored:
raise engine_client.dead_error
base_id = self._base_request_id(raw_request, default=request.request_id)
request_id = f"decision-{base_id}"
try:
questions = parse_questions(request, self.limits)
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine_client.check_admission(len(questions))
reads = await self.strategy.read(
questions,
request.instructions,
state_text(request.state),
request_id=request_id,
chat_template_kwargs=request.chat_template_kwargs,
lora_request=lora_request,
priority=request.priority,
)
except StructuredDecisionError as e:
return self.create_error_response(e)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
answers: dict[str, dict[str, Any]] = {}
diagnostics: dict[str, QuestionDiagnostics] = {}
for q, read in zip(questions, reads):
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
diagnostics[q.id] = QuestionDiagnostics(
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
)
return StructuredDecisionResponse(
id=request_id,
model=self.models.model_name(lora_request),
answers=answers,
usage=DecisionUsage(
input_tokens=sum(r.input_tokens for r in reads),
output_tokens=sum(r.output_tokens for r in reads),
),
diagnostics=diagnostics,
)
@@ -0,0 +1,235 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Read strategies: how a model is asked for label probabilities.
Only models in NEXT_TOKEN_ARCHITECTURES are currently supported.
"""
import math
from abc import ABC, abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.generate.label_reads import next_token_label_reads
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers.inputs.preprocess import extract_prompt_components
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
from .protocol import ReadPromptRequest
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
@dataclass
class ReadContext:
engine_client: EngineClient
online_renderer: OnlineRenderer
chat_template: str | None
chat_template_content_format: ChatTemplateContentFormatOption
default_chat_template_kwargs: dict[str, Any]
@dataclass(frozen=True)
class DecisionLimits:
max_questions: int
max_options: int
@dataclass
class QuestionRead:
probs: list[float]
label_mass: float
argmax_is_label: bool
input_tokens: int
output_tokens: int
class ReadStrategy(ABC):
def __init__(self, context: ReadContext):
self.context = context
@abstractmethod
def limits(self) -> DecisionLimits: ...
@abstractmethod
async def read(
self,
questions: list[Question],
instructions: str | None,
state: str,
*,
request_id: str,
chat_template_kwargs: dict[str, Any] | None,
lora_request: LoRARequest | None,
priority: int,
) -> list[QuestionRead]:
"""One read per question, in order."""
def reply_label_ids(
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
) -> tuple[list[int], list[int]]:
"""The prompt's ids after its last added token, and the token each of LABELS
adds after them as the first token of the reply. Raises ValueError if a
label is not one distinct token there."""
added = set(tokenizer.get_added_vocab().values())
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
tail = list(prompt_ids[start:])
text = tokenizer.decode(tail)
ids: list[int] = []
for label in LABELS:
extended = tokenizer.encode(text + label, add_special_tokens=False)
if extended[:-1] != tail or extended[-1] in ids:
raise ValueError(
f"label {label!r} is not one distinct token after this model's "
"chat prompt"
)
ids.append(extended[-1])
return tail, ids
class NextTokenStrategy(ReadStrategy):
"""Autoregressive models. Each question is one request: the state, then the
question with its labeled options, and the logprobs of the label tokens as
the reply's first token. The requests share the state. With prefix caching,
it is prefilled once."""
def __init__(self, context: ReadContext):
super().__init__(context)
tokenizer = context.online_renderer.renderer.get_tokenizer()
probe = tokenizer.apply_chat_template(
[{"role": "user", "content": "x"}],
**{
**context.default_chat_template_kwargs,
"chat_template": context.chat_template,
"add_generation_prompt": True,
"tokenize": True,
"return_dict": False,
"enable_thinking": False,
},
)
if isinstance(probe, str):
probe = tokenizer.encode(probe, add_special_tokens=False)
# Every prompt ends with the same generation prompt, so the labels'
# tokens are the same for every question.
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
def limits(self) -> DecisionLimits:
return DecisionLimits(max_questions=64, max_options=len(LABELS))
def _read_request(
self, chat_template_kwargs: dict[str, Any] | None
) -> ReadPromptRequest:
if (chat_template_kwargs or {}).get("enable_thinking"):
raise StructuredDecisionError(
"a read is the reply's first token, so thinking must be off"
)
return ReadPromptRequest(chat_template_kwargs=chat_template_kwargs)
async def _render(
self, read_request: ReadPromptRequest, messages: list[dict[str, Any]]
) -> tuple[EngineInput, list[int]]:
ctx = self.context
_, (engine_input,) = await ctx.online_renderer.preprocess_chat(
read_request,
messages,
default_template=ctx.chat_template,
default_template_content_format=ctx.chat_template_content_format,
default_template_kwargs=ctx.default_chat_template_kwargs,
)
prompt_ids = extract_prompt_components(
ctx.engine_client.model_config, engine_input
).token_ids
return engine_input, list(prompt_ids or [])
async def read(
self,
questions: list[Question],
instructions: str | None,
state: str,
*,
request_id: str,
chat_template_kwargs: dict[str, Any] | None,
lora_request: LoRARequest | None,
priority: int,
) -> list[QuestionRead]:
ctx = self.context
read_request = self._read_request(chat_template_kwargs)
slots, engine_inputs = [], []
for q in questions:
messages = [{"role": "user", "content": f"{state}\n\n{q.type.prompt(q)}"}]
if instructions:
messages.insert(0, {"role": "system", "content": instructions})
engine_input, prompt_ids = await self._render(read_request, messages)
if prompt_ids[len(prompt_ids) - len(self.tail) :] != self.tail:
raise StructuredDecisionError(
"these chat options end the prompt differently, so the "
"labels' tokens are unknown"
)
slots.append(self.label_ids[: len(q.labels)])
engine_inputs.append(engine_input)
label_reads = await next_token_label_reads(
ctx.engine_client,
engine_inputs,
[
SamplingParams(max_tokens=1, temperature=0.0, logprob_token_ids=ids)
for ids in slots
],
request_id,
lora_request=lora_request,
priority=priority,
)
reads = []
for ids, label_read in zip(slots, label_reads):
output = label_read.result.outputs[0]
reads.append(
QuestionRead(
probs=label_softmax(label_read.logprobs),
label_mass=sum(math.exp(lp) for lp in label_read.logprobs),
argmax_is_label=bool(output.token_ids)
and output.token_ids[0] in ids,
input_tokens=len(label_read.result.prompt_token_ids or ()),
output_tokens=len(output.token_ids),
)
)
return reads
NEXT_TOKEN_ARCHITECTURES = frozenset(
{
"Qwen3ForCausalLM",
"Qwen3_5ForConditionalGeneration",
"Qwen3_5MoeForConditionalGeneration",
}
)
# label_mass sums the labels' full-vocabulary probabilities. The logprobs modes
# return those probabilities. The logits modes return raw logits instead.
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
"""Raises ValueError when the model cannot serve structured decisions. The
server opted in with --enable-structured-decisions, so startup fails."""
if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES:
raise ValueError(
"--enable-structured-decisions does not support "
f"{model_config.architecture}. Supported architectures: "
f"{sorted(NEXT_TOKEN_ARCHITECTURES)}"
)
if model_config.logprobs_mode not in LOGPROBS_MODES:
raise ValueError(
"--enable-structured-decisions needs --logprobs-mode raw_logprobs or "
f"processed_logprobs, not {model_config.logprobs_mode}"
)
return NextTokenStrategy