[Feat] Structured decisions endpoint (/v1/systemone) (#59299)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
Lucas Wilkinson
Francisco Javier Arceo
Chauncey
parent
273e099693
commit
d33872dfeb
@@ -0,0 +1,105 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.question_types import (
|
||||
QUESTION_TYPES,
|
||||
Option,
|
||||
Question,
|
||||
QuestionType,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
register_question_type,
|
||||
)
|
||||
|
||||
|
||||
def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"choice",
|
||||
instructions,
|
||||
criteria or {"billing": "money", "outage": None, "other": None},
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def test_choice_labels_follow_option_order():
|
||||
q = choice()
|
||||
assert q.labels == ("A", "B", "C")
|
||||
assert [a.name for a in q.options] == ["billing", "outage", "other"]
|
||||
assert q.options[0].description == "money"
|
||||
wide = choice(criteria={str(i): None for i in range(26)})
|
||||
assert wide.labels[-1] == "Z"
|
||||
|
||||
|
||||
def test_choice_prompt():
|
||||
q = choice()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Which team?\n"
|
||||
"A: billing - money\n"
|
||||
"B: outage\n"
|
||||
"C: other\n"
|
||||
"Answer with the letter of one option only."
|
||||
)
|
||||
|
||||
|
||||
def test_choice_answer_shape():
|
||||
q = choice()
|
||||
answer = q.type.answer(q, [0.2, 0.7, 0.1], 0.5)
|
||||
assert answer == {
|
||||
"type": "choice",
|
||||
"choice": "outage",
|
||||
"probabilities": {"billing": 0.2, "outage": 0.7, "other": 0.1},
|
||||
"confidence": 0.35,
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"qid,type_name,criteria,match",
|
||||
[
|
||||
("", "choice", {"x": None, "y": None}, "non-empty"),
|
||||
("q", "nope", {"x": None, "y": None}, "unknown question type"),
|
||||
("q", "choice", ["x", "y"], "must map option names"),
|
||||
],
|
||||
)
|
||||
def test_build_question_rejects(qid, type_name, criteria, match):
|
||||
with pytest.raises(StructuredDecisionError, match=match):
|
||||
build_question(qid, type_name, "", criteria, 128)
|
||||
|
||||
|
||||
def test_one_option_choice():
|
||||
q = choice(criteria={"only": None})
|
||||
assert q.labels == ("A",)
|
||||
assert q.type.answer(q, [1.0], 0.4)["confidence"] == 0.4
|
||||
|
||||
|
||||
def test_option_limit():
|
||||
with pytest.raises(StructuredDecisionError, match="at most 2 options"):
|
||||
choice(criteria={"x": None, "y": None, "z": None}, max_options=2)
|
||||
with pytest.raises(StructuredDecisionError, match="at most 26 options"):
|
||||
choice(criteria={str(i): None for i in range(27)})
|
||||
|
||||
|
||||
def test_registered_type_plugs_in():
|
||||
class BinaryQuestion(QuestionType):
|
||||
name = "test_binary"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
return [Option("yes"), Option("no")]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {"type": self.name, "yes": probs[0]}
|
||||
|
||||
register_question_type(BinaryQuestion)
|
||||
try:
|
||||
q = build_question("ok", "test_binary", "Is it fine?", None, 128)
|
||||
assert q.labels == ("A", "B")
|
||||
assert q.type.answer(q, [0.9, 0.1], 1.0) == {"type": "test_binary", "yes": 0.9}
|
||||
with pytest.raises(ValueError, match="already registered"):
|
||||
register_question_type(BinaryQuestion)
|
||||
finally:
|
||||
del QUESTION_TYPES["test_binary"]
|
||||
@@ -0,0 +1,65 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from types import SimpleNamespace
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.entrypoints.generate.structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
from vllm.entrypoints.generate.structured_decisions.question_types import LABELS
|
||||
from vllm.entrypoints.generate.structured_decisions.strategies import (
|
||||
NextTokenStrategy,
|
||||
reply_label_ids,
|
||||
select_read_strategy,
|
||||
)
|
||||
|
||||
|
||||
def model(architecture: str, logprobs_mode: str = "raw_logprobs") -> Any:
|
||||
return SimpleNamespace(architecture=architecture, logprobs_mode=logprobs_mode)
|
||||
|
||||
|
||||
def test_strategy_selection():
|
||||
qwen = "Qwen3ForCausalLM"
|
||||
assert select_read_strategy(model(qwen)) is NextTokenStrategy
|
||||
assert select_read_strategy(model(qwen, "processed_logprobs")) is NextTokenStrategy
|
||||
with pytest.raises(ValueError, match="does not support LlamaForCausalLM"):
|
||||
select_read_strategy(model("LlamaForCausalLM"))
|
||||
with pytest.raises(ValueError, match="not raw_logits"):
|
||||
select_read_strategy(model(qwen, "raw_logits"))
|
||||
|
||||
|
||||
def test_route_needs_the_flag():
|
||||
for enabled in (False, True):
|
||||
app = FastAPI()
|
||||
app.state.args = SimpleNamespace(enable_structured_decisions=enabled)
|
||||
register_structured_decisions_api_router(app)
|
||||
paths = {getattr(route, "path", None) for route in app.routes}
|
||||
assert ("/v1/systemone" in paths) == enabled
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def qwen():
|
||||
from transformers import AutoTokenizer
|
||||
|
||||
tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen3-0.6B")
|
||||
prompt_ids = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "state"}],
|
||||
add_generation_prompt=True,
|
||||
enable_thinking=False,
|
||||
tokenize=True,
|
||||
return_dict=False,
|
||||
)
|
||||
return tokenizer, prompt_ids
|
||||
|
||||
|
||||
def test_labels_start_the_reply(qwen):
|
||||
tokenizer, prompt_ids = qwen
|
||||
tail, ids = reply_label_ids(tokenizer, prompt_ids)
|
||||
assert tokenizer.decode(tail) == "\n\n"
|
||||
assert [tokenizer.decode([i]) for i in ids] == list(LABELS)
|
||||
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
|
||||
with pytest.raises(ValueError, match="not one distinct token"):
|
||||
reply_label_ids(tokenizer, tokenizer.encode("team:"))
|
||||
@@ -0,0 +1,217 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import pytest
|
||||
import requests
|
||||
|
||||
from tests.utils import RemoteOpenAIServer
|
||||
|
||||
MODEL_NAME = "Qwen/Qwen3-0.6B"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def server():
|
||||
args = [
|
||||
"--dtype",
|
||||
"bfloat16",
|
||||
"--max-model-len",
|
||||
"1024",
|
||||
"--enforce-eager",
|
||||
"--max-num-seqs",
|
||||
"32",
|
||||
"--enable-prefix-caching",
|
||||
"--enable-structured-decisions",
|
||||
]
|
||||
with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
|
||||
yield remote_server
|
||||
|
||||
|
||||
def post(server, body):
|
||||
return requests.post(server.url_for("v1/systemone"), json=body)
|
||||
|
||||
|
||||
def choice(instructions, *names):
|
||||
return {
|
||||
"type": "choice",
|
||||
"instructions": instructions,
|
||||
"criteria": dict.fromkeys(names),
|
||||
}
|
||||
|
||||
|
||||
COPY = {
|
||||
"team": choice(
|
||||
"Which team does the message name?", "billing", "shipping", "security"
|
||||
),
|
||||
"lang": choice("Which language does the message name?", "English", "French"),
|
||||
}
|
||||
|
||||
|
||||
def test_choice_decision(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"team": "billing", "language": "French"},
|
||||
"questions": COPY,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
assert body["object"] == "structured_decision"
|
||||
assert list(body["answers"]) == ["team", "lang"]
|
||||
team = body["answers"]["team"]
|
||||
assert team["type"] == "choice"
|
||||
assert set(team["probabilities"]) == {"billing", "shipping", "security"}
|
||||
assert sum(team["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert team["confidence"] == pytest.approx(
|
||||
max(team["probabilities"].values()) * body["diagnostics"]["team"]["label_mass"]
|
||||
)
|
||||
for diag in body["diagnostics"].values():
|
||||
assert 0.0 < diag["label_mass"] <= 1.0 + 1e-6
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
assert body["usage"]["input_tokens"] > 0
|
||||
|
||||
|
||||
def test_one_option_choice(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "Team: billing.",
|
||||
"questions": {
|
||||
"team": choice("Which team does the message name?", "billing")
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
team = body["answers"]["team"]
|
||||
assert team["choice"] == "billing"
|
||||
assert team["probabilities"] == {"billing": 1.0}
|
||||
assert team["confidence"] == pytest.approx(
|
||||
body["diagnostics"]["team"]["label_mass"]
|
||||
)
|
||||
|
||||
|
||||
def test_one_option_coin_flip(server):
|
||||
# A fair coin with only "heads" allowed. The answer is forced, so its
|
||||
# confidence must come from the model's probability of the label.
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "I flip a fair coin.",
|
||||
"questions": {"coin": choice("How did the coin land?", "heads")},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
coin = body["answers"]["coin"]
|
||||
assert coin["probabilities"] == {"heads": 1.0}
|
||||
assert coin["confidence"] == pytest.approx(
|
||||
body["diagnostics"]["coin"]["label_mass"]
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"state,questions,expected",
|
||||
[
|
||||
(
|
||||
{"team": "billing", "language": "French"},
|
||||
COPY,
|
||||
{"team": "billing", "lang": "French"},
|
||||
),
|
||||
(
|
||||
"My card was charged twice for one order.",
|
||||
{
|
||||
"card": choice("Does the message mention a card?", "yes", "no"),
|
||||
"mood": choice("How does the customer feel?", "happy", "angry"),
|
||||
},
|
||||
{"card": "yes", "mood": "angry"},
|
||||
),
|
||||
(
|
||||
"Hello.",
|
||||
{
|
||||
"sky": choice(
|
||||
"What color is the sky on a clear day?", "blue", "green", "red"
|
||||
)
|
||||
},
|
||||
{"sky": "blue"},
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_single_read_answers(server, state, questions, expected):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": state,
|
||||
"questions": questions,
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
answers = response.json()["answers"]
|
||||
assert {qid: a["choice"] for qid, a in answers.items()} == expected
|
||||
|
||||
|
||||
def test_option_limit(server):
|
||||
body = {
|
||||
"model": MODEL_NAME,
|
||||
"state": "Pick option 7.",
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
}
|
||||
ok = post(
|
||||
server, {**body, "questions": {"n": choice("Which?", *map(str, range(26)))}}
|
||||
)
|
||||
assert ok.status_code == 200, ok.text
|
||||
assert len(ok.json()["answers"]["n"]["probabilities"]) == 26
|
||||
over = post(
|
||||
server, {**body, "questions": {"n": choice("Which?", *map(str, range(27)))}}
|
||||
)
|
||||
assert over.status_code == 400
|
||||
assert "at most 26 options" in over.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_thinking_is_refused(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": "x",
|
||||
"questions": {"q": choice("Which?", "a", "b")},
|
||||
"chat_template_kwargs": {"enable_thinking": True},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 400
|
||||
assert "thinking must be off" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"questions,match",
|
||||
[
|
||||
(
|
||||
{
|
||||
"q": {
|
||||
"type": "choice",
|
||||
"criteria": {"a": None, "b": None},
|
||||
"depends_on": ["x"],
|
||||
}
|
||||
},
|
||||
"unknown field(s)",
|
||||
),
|
||||
(
|
||||
{
|
||||
f"q{i}": {"type": "choice", "criteria": {"a": None, "b": None}}
|
||||
for i in range(65)
|
||||
},
|
||||
"at most 64",
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_validation_errors(server, questions, match):
|
||||
response = post(server, {"model": MODEL_NAME, "state": "x", "questions": questions})
|
||||
assert response.status_code == 400
|
||||
assert match in response.json()["error"]["message"]
|
||||
Reference in New Issue
Block a user