[Feat] Structured decisions endpoint (/v1/systemone) (#59299)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Matt Mastracci
2026-10-08 14:48:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Lucas Wilkinson Lucas Wilkinson Francisco Javier Arceo Chauncey
parent 273e099693
commit d33872dfeb
17 changed files with 1285 additions and 61 deletions
@@ -0,0 +1,105 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from typing import Any
import pytest
from vllm.entrypoints.generate.structured_decisions.question_types import (
QUESTION_TYPES,
Option,
Question,
QuestionType,
StructuredDecisionError,
build_question,
register_question_type,
)
def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=128):
return build_question(
qid,
"choice",
instructions,
criteria or {"billing": "money", "outage": None, "other": None},
max_options,
)
def test_choice_labels_follow_option_order():
q = choice()
assert q.labels == ("A", "B", "C")
assert [a.name for a in q.options] == ["billing", "outage", "other"]
assert q.options[0].description == "money"
wide = choice(criteria={str(i): None for i in range(26)})
assert wide.labels[-1] == "Z"
def test_choice_prompt():
q = choice()
assert q.type.prompt(q) == (
"Question: Which team?\n"
"A: billing - money\n"
"B: outage\n"
"C: other\n"
"Answer with the letter of one option only."
)
def test_choice_answer_shape():
q = choice()
answer = q.type.answer(q, [0.2, 0.7, 0.1], 0.5)
assert answer == {
"type": "choice",
"choice": "outage",
"probabilities": {"billing": 0.2, "outage": 0.7, "other": 0.1},
"confidence": 0.35,
}
@pytest.mark.parametrize(
"qid,type_name,criteria,match",
[
("", "choice", {"x": None, "y": None}, "non-empty"),
("q", "nope", {"x": None, "y": None}, "unknown question type"),
("q", "choice", ["x", "y"], "must map option names"),
],
)
def test_build_question_rejects(qid, type_name, criteria, match):
with pytest.raises(StructuredDecisionError, match=match):
build_question(qid, type_name, "", criteria, 128)
def test_one_option_choice():
q = choice(criteria={"only": None})
assert q.labels == ("A",)
assert q.type.answer(q, [1.0], 0.4)["confidence"] == 0.4
def test_option_limit():
with pytest.raises(StructuredDecisionError, match="at most 2 options"):
choice(criteria={"x": None, "y": None, "z": None}, max_options=2)
with pytest.raises(StructuredDecisionError, match="at most 26 options"):
choice(criteria={str(i): None for i in range(27)})
def test_registered_type_plugs_in():
class BinaryQuestion(QuestionType):
name = "test_binary"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
return [Option("yes"), Option("no")]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {"type": self.name, "yes": probs[0]}
register_question_type(BinaryQuestion)
try:
q = build_question("ok", "test_binary", "Is it fine?", None, 128)
assert q.labels == ("A", "B")
assert q.type.answer(q, [0.9, 0.1], 1.0) == {"type": "test_binary", "yes": 0.9}
with pytest.raises(ValueError, match="already registered"):
register_question_type(BinaryQuestion)
finally:
del QUESTION_TYPES["test_binary"]
@@ -0,0 +1,65 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from types import SimpleNamespace
from typing import Any
import pytest
from fastapi import FastAPI
from vllm.entrypoints.generate.structured_decisions.api_router import (
register_structured_decisions_api_router,
)
from vllm.entrypoints.generate.structured_decisions.question_types import LABELS
from vllm.entrypoints.generate.structured_decisions.strategies import (
NextTokenStrategy,
reply_label_ids,
select_read_strategy,
)
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
return SimpleNamespace(architecture=architecture, logprobs_mode=logprobs_mode)
def test_strategy_selection():
qwen = "Qwen3ForCausalLM"
assert select_read_strategy(model(qwen)) is NextTokenStrategy
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
select_read_strategy(model("LlamaForCausalLM"))
with pytest.raises(ValueError, match="not raw_logits"):
select_read_strategy(model(qwen, "raw_logits"))
def test_route_needs_the_flag():
for enabled in (False, True):
app = FastAPI()
app.state.args = SimpleNamespace(enable_structured_decisions=enabled)
register_structured_decisions_api_router(app)
paths = {getattr(route, "path", None) for route in app.routes}
assert ("/v1/systemone" in paths) == enabled
@pytest.fixture(scope="module")
def qwen():
from transformers import AutoTokenizer
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
prompt_ids = tokenizer.apply_chat_template(
[{"role": "user", "content": "state"}],
add_generation_prompt=True,
enable_thinking=False,
tokenize=True,
return_dict=False,
)
return tokenizer, prompt_ids
def test_labels_start_the_reply(qwen):
tokenizer, prompt_ids = qwen
tail, ids = reply_label_ids(tokenizer, prompt_ids)
assert tokenizer.decode(tail) == "\n\n"
assert [tokenizer.decode([i]) for i in ids] == list(LABELS)
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
with pytest.raises(ValueError, match="not one distinct token"):
reply_label_ids(tokenizer, tokenizer.encode("team:"))
@@ -0,0 +1,217 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import pytest
import requests
from tests.utils import RemoteOpenAIServer
MODEL_NAME = "Qwen/Qwen3-0.6B"
@pytest.fixture(scope="module")
def server():
args = [
"--dtype",
"bfloat16",
"--max-model-len",
"1024",
"--enforce-eager",
"--max-num-seqs",
"32",
"--enable-prefix-caching",
"--enable-structured-decisions",
]
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
yield remote_server
def post(server, body):
return requests.post(server.url_for("v1/systemone"), json=body)
def choice(instructions, *names):
return {
"type": "choice",
"instructions": instructions,
"criteria": dict.fromkeys(names),
}
COPY = {
"team": choice(
"Which team does the message name?", "billing", "shipping", "security"
),
"lang": choice("Which language does the message name?", "English", "French"),
}
def test_choice_decision(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": {"team": "billing", "language": "French"},
"questions": COPY,
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
assert body["object"] == "structured_decision"
assert list(body["answers"]) == ["team", "lang"]
team = body["answers"]["team"]
assert team["type"] == "choice"
assert set(team["probabilities"]) == {"billing", "shipping", "security"}
assert sum(team["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
assert team["confidence"] == pytest.approx(
max(team["probabilities"].values()) * body["diagnostics"]["team"]["label_mass"]
)
for diag in body["diagnostics"].values():
assert 0.0 < diag["label_mass"] <= 1.0 + 1e-6
assert body["usage"]["output_tokens"] == 2
assert body["usage"]["input_tokens"] > 0
def test_one_option_choice(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": "Team: billing.",
"questions": {
"team": choice("Which team does the message name?", "billing")
},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
team = body["answers"]["team"]
assert team["choice"] == "billing"
assert team["probabilities"] == {"billing": 1.0}
assert team["confidence"] == pytest.approx(
body["diagnostics"]["team"]["label_mass"]
)
def test_one_option_coin_flip(server):
# A fair coin with only "heads" allowed. The answer is forced, so its
# confidence must come from the model's probability of the label.
response = post(
server,
{
"model": MODEL_NAME,
"state": "I flip a fair coin.",
"questions": {"coin": choice("How did the coin land?", "heads")},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
coin = body["answers"]["coin"]
assert coin["probabilities"] == {"heads": 1.0}
assert coin["confidence"] == pytest.approx(
body["diagnostics"]["coin"]["label_mass"]
)
@pytest.mark.parametrize(
"state,questions,expected",
[
(
{"team": "billing", "language": "French"},
COPY,
{"team": "billing", "lang": "French"},
),
(
"My card was charged twice for one order.",
{
"card": choice("Does the message mention a card?", "yes", "no"),
"mood": choice("How does the customer feel?", "happy", "angry"),
},
{"card": "yes", "mood": "angry"},
),
(
"Hello.",
{
"sky": choice(
"What color is the sky on a clear day?", "blue", "green", "red"
)
},
{"sky": "blue"},
),
],
)
def test_single_read_answers(server, state, questions, expected):
response = post(
server,
{
"model": MODEL_NAME,
"state": state,
"questions": questions,
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
answers = response.json()["answers"]
assert {qid: a["choice"] for qid, a in answers.items()} == expected
def test_option_limit(server):
body = {
"model": MODEL_NAME,
"state": "Pick option 7.",
"chat_template_kwargs": {"enable_thinking": False},
}
ok = post(
server, {**body, "questions": {"n": choice("Which?", *map(str, range(26)))}}
)
assert ok.status_code == 200, ok.text
assert len(ok.json()["answers"]["n"]["probabilities"]) == 26
over = post(
server, {**body, "questions": {"n": choice("Which?", *map(str, range(27)))}}
)
assert over.status_code == 400
assert "at most 26 options" in over.json()["error"]["message"]
def test_thinking_is_refused(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": "x",
"questions": {"q": choice("Which?", "a", "b")},
"chat_template_kwargs": {"enable_thinking": True},
},
)
assert response.status_code == 400
assert "thinking must be off" in response.json()["error"]["message"]
@pytest.mark.parametrize(
"questions,match",
[
(
{
"q": {
"type": "choice",
"criteria": {"a": None, "b": None},
"depends_on": ["x"],
}
},
"unknown field(s)",
),
(
{
f"q{i}": {"type": "choice", "criteria": {"a": None, "b": None}}
for i in range(65)
},
"at most 64",
),
],
)
def test_validation_errors(server, questions, match):
response = post(server, {"model": MODEL_NAME, "state": "x", "questions": questions})
assert response.status_code == 400
assert match in response.json()["error"]["message"]