[Frontend] structured decisions: noul and score question types (#60696)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Co-authored-by: DeepSeek <noreply@deepseek.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
DeepSeek
Claude Opus 5.5
Misha Goin
parent
88a419649e
commit
8cbd5d0300
@@ -25,6 +25,20 @@ def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=
|
||||
)
|
||||
|
||||
|
||||
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
|
||||
return build_question(qid, "noul", instructions, criteria, 128)
|
||||
|
||||
|
||||
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"score",
|
||||
instructions,
|
||||
criteria or ["calm", "annoyed", "furious"],
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def test_choice_labels_follow_option_order():
|
||||
q = choice()
|
||||
assert q.labels == ("A", "B", "C")
|
||||
@@ -62,6 +76,10 @@ def test_choice_answer_shape():
|
||||
("", "choice", {"x": None, "y": None}, "non-empty"),
|
||||
("q", "nope", {"x": None, "y": None}, "unknown question type"),
|
||||
("q", "choice", ["x", "y"], "must map option names"),
|
||||
("q", "noul", ["yes", "no"], "must be an object"),
|
||||
("q", "noul", {"maybe": "same day"}, "must be an object"),
|
||||
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
|
||||
("q", "score", ["calm"], "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_build_question_rejects(qid, type_name, criteria, match):
|
||||
@@ -82,6 +100,62 @@ def test_option_limit():
|
||||
choice(criteria={str(i): None for i in range(27)})
|
||||
|
||||
|
||||
def test_noul_labels_and_prompt():
|
||||
q = noul()
|
||||
assert q.labels == ("yes", "no")
|
||||
assert [a.name for a in q.options] == ["yes", "no"]
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Reply within the hour?\nAnswer with yes or no only."
|
||||
)
|
||||
described = noul(criteria={"true": "same day", "false": "later"})
|
||||
assert described.type.prompt(described) == (
|
||||
"Question: Reply within the hour?\n"
|
||||
"yes: same day\n"
|
||||
"no: later\n"
|
||||
"Answer with yes or no only."
|
||||
)
|
||||
|
||||
|
||||
def test_noul_answer_shape():
|
||||
q = noul()
|
||||
answer = q.type.answer(q, [0.8, 0.2], 0.5)
|
||||
assert answer == {
|
||||
"type": "noul",
|
||||
"noul": 0.8,
|
||||
"probabilities": {"yes": 0.8, "no": 0.2},
|
||||
"confidence": 0.4,
|
||||
}
|
||||
|
||||
|
||||
def test_score_labels_are_levels():
|
||||
assert score().labels == ("0", "1", "2")
|
||||
with pytest.raises(StructuredDecisionError, match="at most 10"):
|
||||
score(criteria=[str(i) for i in range(11)])
|
||||
|
||||
|
||||
def test_score_prompt():
|
||||
q = score()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: How angry?\n"
|
||||
"0: calm\n"
|
||||
"1: annoyed\n"
|
||||
"2: furious\n"
|
||||
"Answer with the number of one level only."
|
||||
)
|
||||
|
||||
|
||||
def test_score_answer_shape():
|
||||
q = score()
|
||||
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
|
||||
assert answer == {
|
||||
"type": "score",
|
||||
"score": 0.75,
|
||||
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
|
||||
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
|
||||
"confidence": 0.2,
|
||||
}
|
||||
|
||||
|
||||
def test_registered_type_plugs_in():
|
||||
class BinaryQuestion(QuestionType):
|
||||
name = "test_binary"
|
||||
|
||||
@@ -87,6 +87,9 @@ def test_labels_start_the_reply(qwen):
|
||||
tail, ids = reply_label_ids(tokenizer, prompt_ids)
|
||||
assert tokenizer.decode(tail) == "\n\n"
|
||||
assert [tokenizer.decode([i]) for i in ids] == list(LABELS)
|
||||
# A noul's and a score's labels are one token here too.
|
||||
_, ids = reply_label_ids(tokenizer, prompt_ids, ("yes", "no", *"0123456789"))
|
||||
assert len(set(ids)) == 12
|
||||
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
|
||||
with pytest.raises(ValueError, match="not one distinct token"):
|
||||
reply_label_ids(tokenizer, tokenizer.encode("team:"))
|
||||
|
||||
@@ -188,6 +188,48 @@ def test_thinking_is_refused(server):
|
||||
assert "thinking must be off" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_noul_and_score(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"ticket": "My card was charged twice for one order."},
|
||||
"questions": {
|
||||
"urgent": {"type": "noul", "instructions": "Is money moving?"},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset is the customer?",
|
||||
"criteria": ["calm", "annoyed", "furious"],
|
||||
},
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
urgent = body["answers"]["urgent"]
|
||||
assert urgent["type"] == "noul"
|
||||
assert set(urgent["probabilities"]) == {"yes", "no"}
|
||||
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
|
||||
pytest.approx(1.0, abs=1e-4)
|
||||
)
|
||||
assert urgent["noul"] == urgent["probabilities"]["yes"]
|
||||
assert urgent["confidence"] == pytest.approx(
|
||||
max(urgent["probabilities"].values())
|
||||
* body["diagnostics"]["urgent"]["label_mass"]
|
||||
)
|
||||
tone = body["answers"]["tone"]
|
||||
assert tone["type"] == "score"
|
||||
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
|
||||
assert set(tone["probabilities"]) == {"0", "1", "2"}
|
||||
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert tone["score"] == pytest.approx(
|
||||
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
|
||||
)
|
||||
assert 0 <= tone["score"] <= 2
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"questions,match",
|
||||
[
|
||||
@@ -208,6 +250,8 @@ def test_thinking_is_refused(server):
|
||||
},
|
||||
"at most 64",
|
||||
),
|
||||
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
|
||||
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_validation_errors(server, questions, match):
|
||||
|
||||
Reference in New Issue
Block a user