[Frontend] structured decisions: noul and score question types (#60696)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Co-authored-by: DeepSeek <noreply@deepseek.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Matt Mastracci
2026-10-09 04:48:16 +00:00
committed by GitHub
co-authored by DeepSeek Claude Opus 5.5 Misha Goin
parent 88a419649e
commit 8cbd5d0300
6 changed files with 270 additions and 27 deletions
@@ -11,8 +11,8 @@ log probabilities.
## How it works
1. Each question is one read. The user message is the state, followed by the
question and its options, labeled `A`, `B`, `C`, ... in order, and
"Answer with the letter of one option only."
question and its options with the type's labels, and an instruction to
answer with one label.
2. The read is the reply's first token. Each label must be one token there,
or the request returns 400.
3. The read returns the logprobs of that question's label tokens. Softmax over
@@ -31,8 +31,11 @@ question prefills only its own text, then reads one token.
| type | criteria | answer |
| --- | --- | --- |
| `choice` | map of option name to a description or `null` | `choice`, `probabilities` by option name, `confidence` |
| `noul` | `null`, or an object with optional `true` and `false` descriptions | `noul` (probability of yes), `probabilities`, `confidence` |
| `score` | ordered list of level names | `score` (expected 0-indexed level), `legend`, `probabilities` by level index, `confidence` |
A choice has at least one option. A question id is a non-empty string.
A choice has at least one option, a score at least two levels. A question id
is a non-empty string.
## Example
@@ -51,6 +54,16 @@ curl -s http://localhost:8000/v1/systemone \
"shipping": "deliveries",
"security": "account access"
}
},
"urgent": {
"type": "noul",
"instructions": "Does the customer need a reply within the hour?",
"criteria": {"true": "the account is locked or money is moving"}
},
"tone": {
"type": "score",
"instructions": "How upset is the customer?",
"criteria": ["calm", "annoyed", "furious"]
}
},
"chat_template_kwargs": {"enable_thinking": false}
@@ -71,11 +84,26 @@ Response, with probabilities that depend on the model:
"choice": "billing",
"probabilities": {"billing": 0.97, "shipping": 0.01, "security": 0.02},
"confidence": 0.90
},
"urgent": {
"type": "noul",
"noul": 0.02,
"probabilities": {"yes": 0.02, "no": 0.98},
"confidence": 0.95
},
"tone": {
"type": "score",
"score": 0.6,
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
"probabilities": {"0": 0.7, "1": 0.2, "2": 0.1},
"confidence": 0.65
}
},
"usage": {"input_tokens": 142, "output_tokens": 1},
"usage": {"input_tokens": 142, "output_tokens": 3},
"diagnostics": {
"team": {"label_mass": 0.93, "argmax_is_label": true}
"team": {"label_mass": 0.93, "argmax_is_label": true},
"urgent": {"label_mass": 0.98, "argmax_is_label": true},
"tone": {"label_mass": 0.91, "argmax_is_label": true}
}
}
```
@@ -87,8 +115,9 @@ whole vocabulary: its share of the labels times `label_mass`.
## Labels
Options are labeled `A` to `Z` in the order the request lists them, so a
`choice` has at most 26 options.
Labels follow the type: a `choice` labels its options `A` to `Z` in the order
the request lists them, a `noul` labels its options `yes` and `no`, and a
`score` labels its levels `0` to `9`, so a level's label is its score.
## Limits
@@ -96,6 +125,7 @@ Options are labeled `A` to `Z` in the order the request lists them, so a
| --- | --- |
| questions per request | 64 |
| options per `choice` | 26 |
| levels per `score` | 10 |
## Request fields
@@ -25,6 +25,20 @@ def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=
)
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
return build_question(qid, "noul", instructions, criteria, 128)
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
return build_question(
qid,
"score",
instructions,
criteria or ["calm", "annoyed", "furious"],
max_options,
)
def test_choice_labels_follow_option_order():
q = choice()
assert q.labels == ("A", "B", "C")
@@ -62,6 +76,10 @@ def test_choice_answer_shape():
("", "choice", {"x": None, "y": None}, "non-empty"),
("q", "nope", {"x": None, "y": None}, "unknown question type"),
("q", "choice", ["x", "y"], "must map option names"),
("q", "noul", ["yes", "no"], "must be an object"),
("q", "noul", {"maybe": "same day"}, "must be an object"),
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
("q", "score", ["calm"], "ordered list of levels"),
],
)
def test_build_question_rejects(qid, type_name, criteria, match):
@@ -82,6 +100,62 @@ def test_option_limit():
choice(criteria={str(i): None for i in range(27)})
def test_noul_labels_and_prompt():
q = noul()
assert q.labels == ("yes", "no")
assert [a.name for a in q.options] == ["yes", "no"]
assert q.type.prompt(q) == (
"Question: Reply within the hour?\nAnswer with yes or no only."
)
described = noul(criteria={"true": "same day", "false": "later"})
assert described.type.prompt(described) == (
"Question: Reply within the hour?\n"
"yes: same day\n"
"no: later\n"
"Answer with yes or no only."
)
def test_noul_answer_shape():
q = noul()
answer = q.type.answer(q, [0.8, 0.2], 0.5)
assert answer == {
"type": "noul",
"noul": 0.8,
"probabilities": {"yes": 0.8, "no": 0.2},
"confidence": 0.4,
}
def test_score_labels_are_levels():
assert score().labels == ("0", "1", "2")
with pytest.raises(StructuredDecisionError, match="at most 10"):
score(criteria=[str(i) for i in range(11)])
def test_score_prompt():
q = score()
assert q.type.prompt(q) == (
"Question: How angry?\n"
"0: calm\n"
"1: annoyed\n"
"2: furious\n"
"Answer with the number of one level only."
)
def test_score_answer_shape():
q = score()
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
assert answer == {
"type": "score",
"score": 0.75,
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
"confidence": 0.2,
}
def test_registered_type_plugs_in():
class BinaryQuestion(QuestionType):
name = "test_binary"
@@ -87,6 +87,9 @@ def test_labels_start_the_reply(qwen):
tail, ids = reply_label_ids(tokenizer, prompt_ids)
assert tokenizer.decode(tail) == "\n\n"
assert [tokenizer.decode([i]) for i in ids] == list(LABELS)
# A noul's and a score's labels are one token here too.
_, ids = reply_label_ids(tokenizer, prompt_ids, ("yes", "no", *"0123456789"))
assert len(set(ids)) == 12
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
with pytest.raises(ValueError, match="not one distinct token"):
reply_label_ids(tokenizer, tokenizer.encode("team:"))
@@ -188,6 +188,48 @@ def test_thinking_is_refused(server):
assert "thinking must be off" in response.json()["error"]["message"]
def test_noul_and_score(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": {"ticket": "My card was charged twice for one order."},
"questions": {
"urgent": {"type": "noul", "instructions": "Is money moving?"},
"tone": {
"type": "score",
"instructions": "How upset is the customer?",
"criteria": ["calm", "annoyed", "furious"],
},
},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
urgent = body["answers"]["urgent"]
assert urgent["type"] == "noul"
assert set(urgent["probabilities"]) == {"yes", "no"}
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
pytest.approx(1.0, abs=1e-4)
)
assert urgent["noul"] == urgent["probabilities"]["yes"]
assert urgent["confidence"] == pytest.approx(
max(urgent["probabilities"].values())
* body["diagnostics"]["urgent"]["label_mass"]
)
tone = body["answers"]["tone"]
assert tone["type"] == "score"
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
assert set(tone["probabilities"]) == {"0", "1", "2"}
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
assert tone["score"] == pytest.approx(
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
)
assert 0 <= tone["score"] <= 2
assert body["usage"]["output_tokens"] == 2
@pytest.mark.parametrize(
"questions,match",
[
@@ -208,6 +250,8 @@ def test_thinking_is_refused(server):
},
"at most 64",
),
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
],
)
def test_validation_errors(server, questions, match):
@@ -18,8 +18,8 @@ class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: Options are labeled A to Z in order. A single capital letter is one token
#: at the start of a reply for the tokenizers tested; every read checks it.
#: A choice labels its options A to Z in order. A single capital letter is one
#: token at the start of a reply for the tokenizers tested. Every read checks it.
LABELS = tuple(string.ascii_uppercase)
@@ -40,6 +40,9 @@ class Question:
class QuestionType(ABC):
name: ClassVar[str]
#: The labels of a question's options, in option order.
label_set: ClassVar[tuple[str, ...]] = LABELS
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@@ -52,16 +55,22 @@ class QuestionType(ABC):
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def reply_line(self, label: str, option: Option) -> str | None:
"""The option's line in the prompt, or None to leave it out."""
return (
f"{label}: {option.name} - {option.description}"
if option.description
else f"{label}: {option.name}"
)
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(
f"{label}: {o.name} - {o.description}"
if o.description
else f"{label}: {o.name}"
)
lines.append("Answer with the letter of one option only.")
line = self.reply_line(label, o)
if line is not None:
lines.append(line)
lines.append(self.reply_instruction)
return "\n".join(lines)
@@ -100,7 +109,7 @@ def build_question(
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(LABELS))
limit = min(max_options, len(qtype.label_set))
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
@@ -112,7 +121,7 @@ def build_question(
type=qtype,
instructions=instructions,
options=tuple(options),
labels=LABELS[: len(options)],
labels=qtype.label_set[: len(options)],
)
@@ -158,3 +167,74 @@ class ChoiceQuestion(QuestionType):
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@register_question_type
class NoulQuestion(QuestionType):
"""Yes or no. ``criteria`` may describe what true and false mean."""
name = "noul"
label_set = ("yes", "no")
reply_instruction = "Answer with yes or no only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if criteria is not None and (
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
):
raise StructuredDecisionError(
f"question {qid!r}: noul criteria must be an object with true and false"
)
criteria = criteria or {}
true = criteria.get("true")
false = criteria.get("false")
return [
Option("yes", None if true is None else str(true)),
Option("no", None if false is None else str(false)),
]
def reply_line(self, label: str, option: Option) -> str | None:
return f"{label}: {option.description}" if option.description else None
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {
"type": self.name,
"noul": probs[0],
"probabilities": {"yes": probs[0], "no": probs[1]},
# The chosen label's probability over the whole vocabulary, like a
# choice's confidence.
"confidence": max(probs) * label_mass,
}
@register_question_type
class ScoreQuestion(QuestionType):
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
level names, from the first to the last level of the scale."""
name = "score"
# 0-indexed like the answer, so a level's label is its score.
label_set = tuple(string.digits)
reply_instruction = "Answer with the number of one level only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, list) or len(criteria) < 2:
raise StructuredDecisionError(
f"question {qid!r}: score criteria must be an ordered list of levels"
)
return [Option(str(level)) for level in criteria]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
# The expected level, 0-indexed, so a level's index in the request's
# criteria is its value.
"score": sum(i * p for i, p in enumerate(probs)),
"legend": {str(i): o.name for i, o in enumerate(question.options)},
"probabilities": {str(i): p for i, p in enumerate(probs)},
"confidence": probs[top] * label_mass,
}
@@ -23,7 +23,13 @@ from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
from .protocol import ReadPromptRequest
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
from .question_types import (
LABELS,
QUESTION_TYPES,
Question,
StructuredDecisionError,
label_softmax,
)
@dataclass
@@ -73,17 +79,19 @@ class ReadStrategy(ABC):
def reply_label_ids(
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
tokenizer: TokenizerLike,
prompt_ids: Sequence[int],
labels: Sequence[str] = LABELS,
) -> tuple[list[int], list[int]]:
"""The prompt's ids after its last added token, and the token each of LABELS
adds after them as the first token of the reply. Raises ValueError if a
label is not one distinct token there."""
"""The prompt's ids after its last added token, and the token each of
``labels`` adds after them as the first token of the reply. Raises
ValueError if a label is not one distinct token there."""
added = set(tokenizer.get_added_vocab().values())
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
tail = list(prompt_ids[start:])
text = tokenizer.decode(tail)
ids: list[int] = []
for label in LABELS:
for label in labels:
extended = tokenizer.encode(text + label, add_special_tokens=False)
if extended[:-1] != tail or extended[-1] in ids:
raise ValueError(
@@ -116,9 +124,13 @@ class NextTokenStrategy(ReadStrategy):
)
if isinstance(probe, str):
probe = tokenizer.encode(probe, add_special_tokens=False)
# Every prompt ends with the same generation prompt, so the labels'
# tokens are the same for every question.
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
# Every prompt ends with the same generation prompt, so a label's token
# is the same for every question.
self.tail, _ = reply_label_ids(tokenizer, probe, ())
self.label_ids: dict[str, int] = {}
for qtype in QUESTION_TYPES.values():
_, ids = reply_label_ids(tokenizer, probe, qtype.label_set)
self.label_ids.update(zip(qtype.label_set, ids))
def limits(self) -> DecisionLimits:
return DecisionLimits(max_questions=64, max_options=len(LABELS))
@@ -173,7 +185,7 @@ class NextTokenStrategy(ReadStrategy):
"these chat options end the prompt differently, so the "
"labels' tokens are unknown"
)
slots.append(self.label_ids[: len(q.labels)])
slots.append([self.label_ids[label] for label in q.labels])
engine_inputs.append(engine_input)
label_reads = await next_token_label_reads(