[Frontend] structured decisions: noul and score question types (#60696)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Co-authored-by: DeepSeek <noreply@deepseek.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Matt Mastracci
2026-10-09 04:48:16 +00:00
committed by GitHub
co-authored by DeepSeek Claude Opus 5.5 Misha Goin
parent 88a419649e
commit 8cbd5d0300
6 changed files with 270 additions and 27 deletions
@@ -25,6 +25,20 @@ def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=
)
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
return build_question(qid, "noul", instructions, criteria, 128)
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
return build_question(
qid,
"score",
instructions,
criteria or ["calm", "annoyed", "furious"],
max_options,
)
def test_choice_labels_follow_option_order():
q = choice()
assert q.labels == ("A", "B", "C")
@@ -62,6 +76,10 @@ def test_choice_answer_shape():
("", "choice", {"x": None, "y": None}, "non-empty"),
("q", "nope", {"x": None, "y": None}, "unknown question type"),
("q", "choice", ["x", "y"], "must map option names"),
("q", "noul", ["yes", "no"], "must be an object"),
("q", "noul", {"maybe": "same day"}, "must be an object"),
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
("q", "score", ["calm"], "ordered list of levels"),
],
)
def test_build_question_rejects(qid, type_name, criteria, match):
@@ -82,6 +100,62 @@ def test_option_limit():
choice(criteria={str(i): None for i in range(27)})
def test_noul_labels_and_prompt():
q = noul()
assert q.labels == ("yes", "no")
assert [a.name for a in q.options] == ["yes", "no"]
assert q.type.prompt(q) == (
"Question: Reply within the hour?\nAnswer with yes or no only."
)
described = noul(criteria={"true": "same day", "false": "later"})
assert described.type.prompt(described) == (
"Question: Reply within the hour?\n"
"yes: same day\n"
"no: later\n"
"Answer with yes or no only."
)
def test_noul_answer_shape():
q = noul()
answer = q.type.answer(q, [0.8, 0.2], 0.5)
assert answer == {
"type": "noul",
"noul": 0.8,
"probabilities": {"yes": 0.8, "no": 0.2},
"confidence": 0.4,
}
def test_score_labels_are_levels():
assert score().labels == ("0", "1", "2")
with pytest.raises(StructuredDecisionError, match="at most 10"):
score(criteria=[str(i) for i in range(11)])
def test_score_prompt():
q = score()
assert q.type.prompt(q) == (
"Question: How angry?\n"
"0: calm\n"
"1: annoyed\n"
"2: furious\n"
"Answer with the number of one level only."
)
def test_score_answer_shape():
q = score()
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
assert answer == {
"type": "score",
"score": 0.75,
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
"confidence": 0.2,
}
def test_registered_type_plugs_in():
class BinaryQuestion(QuestionType):
name = "test_binary"
@@ -87,6 +87,9 @@ def test_labels_start_the_reply(qwen):
tail, ids = reply_label_ids(tokenizer, prompt_ids)
assert tokenizer.decode(tail) == "\n\n"
assert [tokenizer.decode([i]) for i in ids] == list(LABELS)
# A noul's and a score's labels are one token here too.
_, ids = reply_label_ids(tokenizer, prompt_ids, ("yes", "no", *"0123456789"))
assert len(set(ids)) == 12
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
with pytest.raises(ValueError, match="not one distinct token"):
reply_label_ids(tokenizer, tokenizer.encode("team:"))
@@ -188,6 +188,48 @@ def test_thinking_is_refused(server):
assert "thinking must be off" in response.json()["error"]["message"]
def test_noul_and_score(server):
response = post(
server,
{
"model": MODEL_NAME,
"state": {"ticket": "My card was charged twice for one order."},
"questions": {
"urgent": {"type": "noul", "instructions": "Is money moving?"},
"tone": {
"type": "score",
"instructions": "How upset is the customer?",
"criteria": ["calm", "annoyed", "furious"],
},
},
"chat_template_kwargs": {"enable_thinking": False},
},
)
assert response.status_code == 200, response.text
body = response.json()
urgent = body["answers"]["urgent"]
assert urgent["type"] == "noul"
assert set(urgent["probabilities"]) == {"yes", "no"}
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
pytest.approx(1.0, abs=1e-4)
)
assert urgent["noul"] == urgent["probabilities"]["yes"]
assert urgent["confidence"] == pytest.approx(
max(urgent["probabilities"].values())
* body["diagnostics"]["urgent"]["label_mass"]
)
tone = body["answers"]["tone"]
assert tone["type"] == "score"
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
assert set(tone["probabilities"]) == {"0", "1", "2"}
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
assert tone["score"] == pytest.approx(
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
)
assert 0 <= tone["score"] <= 2
assert body["usage"]["output_tokens"] == 2
@pytest.mark.parametrize(
"questions,match",
[
@@ -208,6 +250,8 @@ def test_thinking_is_refused(server):
},
"at most 64",
),
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
],
)
def test_validation_errors(server, questions, match):