[Frontend] structured decisions: noul and score question types (#60696)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Co-authored-by: DeepSeek <noreply@deepseek.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
DeepSeek
Claude Opus 5.5
Misha Goin
parent
88a419649e
commit
8cbd5d0300
@@ -11,8 +11,8 @@ log probabilities.
|
||||
## How it works
|
||||
|
||||
1. Each question is one read. The user message is the state, followed by the
|
||||
question and its options, labeled `A`, `B`, `C`, ... in order, and
|
||||
"Answer with the letter of one option only."
|
||||
question and its options with the type's labels, and an instruction to
|
||||
answer with one label.
|
||||
2. The read is the reply's first token. Each label must be one token there,
|
||||
or the request returns 400.
|
||||
3. The read returns the logprobs of that question's label tokens. Softmax over
|
||||
@@ -31,8 +31,11 @@ question prefills only its own text, then reads one token.
|
||||
| type | criteria | answer |
|
||||
| --- | --- | --- |
|
||||
| `choice` | map of option name to a description or `null` | `choice`, `probabilities` by option name, `confidence` |
|
||||
| `noul` | `null`, or an object with optional `true` and `false` descriptions | `noul` (probability of yes), `probabilities`, `confidence` |
|
||||
| `score` | ordered list of level names | `score` (expected 0-indexed level), `legend`, `probabilities` by level index, `confidence` |
|
||||
|
||||
A choice has at least one option. A question id is a non-empty string.
|
||||
A choice has at least one option, a score at least two levels. A question id
|
||||
is a non-empty string.
|
||||
|
||||
## Example
|
||||
|
||||
@@ -51,6 +54,16 @@ curl -s http://localhost:8000/v1/systemone \
|
||||
"shipping": "deliveries",
|
||||
"security": "account access"
|
||||
}
|
||||
},
|
||||
"urgent": {
|
||||
"type": "noul",
|
||||
"instructions": "Does the customer need a reply within the hour?",
|
||||
"criteria": {"true": "the account is locked or money is moving"}
|
||||
},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset is the customer?",
|
||||
"criteria": ["calm", "annoyed", "furious"]
|
||||
}
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": false}
|
||||
@@ -71,11 +84,26 @@ Response, with probabilities that depend on the model:
|
||||
"choice": "billing",
|
||||
"probabilities": {"billing": 0.97, "shipping": 0.01, "security": 0.02},
|
||||
"confidence": 0.90
|
||||
},
|
||||
"urgent": {
|
||||
"type": "noul",
|
||||
"noul": 0.02,
|
||||
"probabilities": {"yes": 0.02, "no": 0.98},
|
||||
"confidence": 0.95
|
||||
},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"score": 0.6,
|
||||
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
|
||||
"probabilities": {"0": 0.7, "1": 0.2, "2": 0.1},
|
||||
"confidence": 0.65
|
||||
}
|
||||
},
|
||||
"usage": {"input_tokens": 142, "output_tokens": 1},
|
||||
"usage": {"input_tokens": 142, "output_tokens": 3},
|
||||
"diagnostics": {
|
||||
"team": {"label_mass": 0.93, "argmax_is_label": true}
|
||||
"team": {"label_mass": 0.93, "argmax_is_label": true},
|
||||
"urgent": {"label_mass": 0.98, "argmax_is_label": true},
|
||||
"tone": {"label_mass": 0.91, "argmax_is_label": true}
|
||||
}
|
||||
}
|
||||
```
|
||||
@@ -87,8 +115,9 @@ whole vocabulary: its share of the labels times `label_mass`.
|
||||
|
||||
## Labels
|
||||
|
||||
Options are labeled `A` to `Z` in the order the request lists them, so a
|
||||
`choice` has at most 26 options.
|
||||
Labels follow the type: a `choice` labels its options `A` to `Z` in the order
|
||||
the request lists them, a `noul` labels its options `yes` and `no`, and a
|
||||
`score` labels its levels `0` to `9`, so a level's label is its score.
|
||||
|
||||
## Limits
|
||||
|
||||
@@ -96,6 +125,7 @@ Options are labeled `A` to `Z` in the order the request lists them, so a
|
||||
| --- | --- |
|
||||
| questions per request | 64 |
|
||||
| options per `choice` | 26 |
|
||||
| levels per `score` | 10 |
|
||||
|
||||
## Request fields
|
||||
|
||||
|
||||
@@ -25,6 +25,20 @@ def choice(qid="bucket", criteria=None, instructions="Which team?", max_options=
|
||||
)
|
||||
|
||||
|
||||
def noul(qid="urgent", criteria=None, instructions="Reply within the hour?"):
|
||||
return build_question(qid, "noul", instructions, criteria, 128)
|
||||
|
||||
|
||||
def score(qid="tone", criteria=None, instructions="How angry?", max_options=128):
|
||||
return build_question(
|
||||
qid,
|
||||
"score",
|
||||
instructions,
|
||||
criteria or ["calm", "annoyed", "furious"],
|
||||
max_options,
|
||||
)
|
||||
|
||||
|
||||
def test_choice_labels_follow_option_order():
|
||||
q = choice()
|
||||
assert q.labels == ("A", "B", "C")
|
||||
@@ -62,6 +76,10 @@ def test_choice_answer_shape():
|
||||
("", "choice", {"x": None, "y": None}, "non-empty"),
|
||||
("q", "nope", {"x": None, "y": None}, "unknown question type"),
|
||||
("q", "choice", ["x", "y"], "must map option names"),
|
||||
("q", "noul", ["yes", "no"], "must be an object"),
|
||||
("q", "noul", {"maybe": "same day"}, "must be an object"),
|
||||
("q", "score", {"0": "calm", "1": "furious"}, "ordered list of levels"),
|
||||
("q", "score", ["calm"], "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_build_question_rejects(qid, type_name, criteria, match):
|
||||
@@ -82,6 +100,62 @@ def test_option_limit():
|
||||
choice(criteria={str(i): None for i in range(27)})
|
||||
|
||||
|
||||
def test_noul_labels_and_prompt():
|
||||
q = noul()
|
||||
assert q.labels == ("yes", "no")
|
||||
assert [a.name for a in q.options] == ["yes", "no"]
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: Reply within the hour?\nAnswer with yes or no only."
|
||||
)
|
||||
described = noul(criteria={"true": "same day", "false": "later"})
|
||||
assert described.type.prompt(described) == (
|
||||
"Question: Reply within the hour?\n"
|
||||
"yes: same day\n"
|
||||
"no: later\n"
|
||||
"Answer with yes or no only."
|
||||
)
|
||||
|
||||
|
||||
def test_noul_answer_shape():
|
||||
q = noul()
|
||||
answer = q.type.answer(q, [0.8, 0.2], 0.5)
|
||||
assert answer == {
|
||||
"type": "noul",
|
||||
"noul": 0.8,
|
||||
"probabilities": {"yes": 0.8, "no": 0.2},
|
||||
"confidence": 0.4,
|
||||
}
|
||||
|
||||
|
||||
def test_score_labels_are_levels():
|
||||
assert score().labels == ("0", "1", "2")
|
||||
with pytest.raises(StructuredDecisionError, match="at most 10"):
|
||||
score(criteria=[str(i) for i in range(11)])
|
||||
|
||||
|
||||
def test_score_prompt():
|
||||
q = score()
|
||||
assert q.type.prompt(q) == (
|
||||
"Question: How angry?\n"
|
||||
"0: calm\n"
|
||||
"1: annoyed\n"
|
||||
"2: furious\n"
|
||||
"Answer with the number of one level only."
|
||||
)
|
||||
|
||||
|
||||
def test_score_answer_shape():
|
||||
q = score()
|
||||
answer = q.type.answer(q, [0.5, 0.25, 0.25], 0.4)
|
||||
assert answer == {
|
||||
"type": "score",
|
||||
"score": 0.75,
|
||||
"legend": {"0": "calm", "1": "annoyed", "2": "furious"},
|
||||
"probabilities": {"0": 0.5, "1": 0.25, "2": 0.25},
|
||||
"confidence": 0.2,
|
||||
}
|
||||
|
||||
|
||||
def test_registered_type_plugs_in():
|
||||
class BinaryQuestion(QuestionType):
|
||||
name = "test_binary"
|
||||
|
||||
@@ -87,6 +87,9 @@ def test_labels_start_the_reply(qwen):
|
||||
tail, ids = reply_label_ids(tokenizer, prompt_ids)
|
||||
assert tokenizer.decode(tail) == "\n\n"
|
||||
assert [tokenizer.decode([i]) for i in ids] == list(LABELS)
|
||||
# A noul's and a score's labels are one token here too.
|
||||
_, ids = reply_label_ids(tokenizer, prompt_ids, ("yes", "no", *"0123456789"))
|
||||
assert len(set(ids)) == 12
|
||||
# After a colon, Qwen writes ":A" as one token, so "A" is not one token.
|
||||
with pytest.raises(ValueError, match="not one distinct token"):
|
||||
reply_label_ids(tokenizer, tokenizer.encode("team:"))
|
||||
|
||||
@@ -188,6 +188,48 @@ def test_thinking_is_refused(server):
|
||||
assert "thinking must be off" in response.json()["error"]["message"]
|
||||
|
||||
|
||||
def test_noul_and_score(server):
|
||||
response = post(
|
||||
server,
|
||||
{
|
||||
"model": MODEL_NAME,
|
||||
"state": {"ticket": "My card was charged twice for one order."},
|
||||
"questions": {
|
||||
"urgent": {"type": "noul", "instructions": "Is money moving?"},
|
||||
"tone": {
|
||||
"type": "score",
|
||||
"instructions": "How upset is the customer?",
|
||||
"criteria": ["calm", "annoyed", "furious"],
|
||||
},
|
||||
},
|
||||
"chat_template_kwargs": {"enable_thinking": False},
|
||||
},
|
||||
)
|
||||
assert response.status_code == 200, response.text
|
||||
body = response.json()
|
||||
urgent = body["answers"]["urgent"]
|
||||
assert urgent["type"] == "noul"
|
||||
assert set(urgent["probabilities"]) == {"yes", "no"}
|
||||
assert urgent["probabilities"]["yes"] + urgent["probabilities"]["no"] == (
|
||||
pytest.approx(1.0, abs=1e-4)
|
||||
)
|
||||
assert urgent["noul"] == urgent["probabilities"]["yes"]
|
||||
assert urgent["confidence"] == pytest.approx(
|
||||
max(urgent["probabilities"].values())
|
||||
* body["diagnostics"]["urgent"]["label_mass"]
|
||||
)
|
||||
tone = body["answers"]["tone"]
|
||||
assert tone["type"] == "score"
|
||||
assert tone["legend"] == {"0": "calm", "1": "annoyed", "2": "furious"}
|
||||
assert set(tone["probabilities"]) == {"0", "1", "2"}
|
||||
assert sum(tone["probabilities"].values()) == pytest.approx(1.0, abs=1e-4)
|
||||
assert tone["score"] == pytest.approx(
|
||||
sum(i * p for i, p in enumerate(tone["probabilities"].values()))
|
||||
)
|
||||
assert 0 <= tone["score"] <= 2
|
||||
assert body["usage"]["output_tokens"] == 2
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"questions,match",
|
||||
[
|
||||
@@ -208,6 +250,8 @@ def test_thinking_is_refused(server):
|
||||
},
|
||||
"at most 64",
|
||||
),
|
||||
({"q": {"type": "noul", "criteria": ["yes", "no"]}}, "must be an object"),
|
||||
({"q": {"type": "score", "criteria": ["calm"]}}, "ordered list of levels"),
|
||||
],
|
||||
)
|
||||
def test_validation_errors(server, questions, match):
|
||||
|
||||
@@ -18,8 +18,8 @@ class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: Options are labeled A to Z in order. A single capital letter is one token
|
||||
#: at the start of a reply for the tokenizers tested; every read checks it.
|
||||
#: A choice labels its options A to Z in order. A single capital letter is one
|
||||
#: token at the start of a reply for the tokenizers tested. Every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@@ -40,6 +40,9 @@ class Question:
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
#: The labels of a question's options, in option order.
|
||||
label_set: ClassVar[tuple[str, ...]] = LABELS
|
||||
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
@@ -52,16 +55,22 @@ class QuestionType(ABC):
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str | None:
|
||||
"""The option's line in the prompt, or None to leave it out."""
|
||||
return (
|
||||
f"{label}: {option.name} - {option.description}"
|
||||
if option.description
|
||||
else f"{label}: {option.name}"
|
||||
)
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(
|
||||
f"{label}: {o.name} - {o.description}"
|
||||
if o.description
|
||||
else f"{label}: {o.name}"
|
||||
)
|
||||
lines.append("Answer with the letter of one option only.")
|
||||
line = self.reply_line(label, o)
|
||||
if line is not None:
|
||||
lines.append(line)
|
||||
lines.append(self.reply_instruction)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -100,7 +109,7 @@ def build_question(
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(LABELS))
|
||||
limit = min(max_options, len(qtype.label_set))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
@@ -112,7 +121,7 @@ def build_question(
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
labels=qtype.label_set[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
@@ -158,3 +167,74 @@ class ChoiceQuestion(QuestionType):
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class NoulQuestion(QuestionType):
|
||||
"""Yes or no. ``criteria`` may describe what true and false mean."""
|
||||
|
||||
name = "noul"
|
||||
label_set = ("yes", "no")
|
||||
reply_instruction = "Answer with yes or no only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if criteria is not None and (
|
||||
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
|
||||
):
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: noul criteria must be an object with true and false"
|
||||
)
|
||||
criteria = criteria or {}
|
||||
true = criteria.get("true")
|
||||
false = criteria.get("false")
|
||||
return [
|
||||
Option("yes", None if true is None else str(true)),
|
||||
Option("no", None if false is None else str(false)),
|
||||
]
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str | None:
|
||||
return f"{label}: {option.description}" if option.description else None
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.name,
|
||||
"noul": probs[0],
|
||||
"probabilities": {"yes": probs[0], "no": probs[1]},
|
||||
# The chosen label's probability over the whole vocabulary, like a
|
||||
# choice's confidence.
|
||||
"confidence": max(probs) * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ScoreQuestion(QuestionType):
|
||||
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
|
||||
level names, from the first to the last level of the scale."""
|
||||
|
||||
name = "score"
|
||||
# 0-indexed like the answer, so a level's label is its score.
|
||||
label_set = tuple(string.digits)
|
||||
reply_instruction = "Answer with the number of one level only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, list) or len(criteria) < 2:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: score criteria must be an ordered list of levels"
|
||||
)
|
||||
return [Option(str(level)) for level in criteria]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
# The expected level, 0-indexed, so a level's index in the request's
|
||||
# criteria is its value.
|
||||
"score": sum(i * p for i, p in enumerate(probs)),
|
||||
"legend": {str(i): o.name for i, o in enumerate(question.options)},
|
||||
"probabilities": {str(i): p for i, p in enumerate(probs)},
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
@@ -23,7 +23,13 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .protocol import ReadPromptRequest
|
||||
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
QUESTION_TYPES,
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
label_softmax,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -73,17 +79,19 @@ class ReadStrategy(ABC):
|
||||
|
||||
|
||||
def reply_label_ids(
|
||||
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
|
||||
tokenizer: TokenizerLike,
|
||||
prompt_ids: Sequence[int],
|
||||
labels: Sequence[str] = LABELS,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""The prompt's ids after its last added token, and the token each of LABELS
|
||||
adds after them as the first token of the reply. Raises ValueError if a
|
||||
label is not one distinct token there."""
|
||||
"""The prompt's ids after its last added token, and the token each of
|
||||
``labels`` adds after them as the first token of the reply. Raises
|
||||
ValueError if a label is not one distinct token there."""
|
||||
added = set(tokenizer.get_added_vocab().values())
|
||||
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
|
||||
tail = list(prompt_ids[start:])
|
||||
text = tokenizer.decode(tail)
|
||||
ids: list[int] = []
|
||||
for label in LABELS:
|
||||
for label in labels:
|
||||
extended = tokenizer.encode(text + label, add_special_tokens=False)
|
||||
if extended[:-1] != tail or extended[-1] in ids:
|
||||
raise ValueError(
|
||||
@@ -116,9 +124,13 @@ class NextTokenStrategy(ReadStrategy):
|
||||
)
|
||||
if isinstance(probe, str):
|
||||
probe = tokenizer.encode(probe, add_special_tokens=False)
|
||||
# Every prompt ends with the same generation prompt, so the labels'
|
||||
# tokens are the same for every question.
|
||||
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
|
||||
# Every prompt ends with the same generation prompt, so a label's token
|
||||
# is the same for every question.
|
||||
self.tail, _ = reply_label_ids(tokenizer, probe, ())
|
||||
self.label_ids: dict[str, int] = {}
|
||||
for qtype in QUESTION_TYPES.values():
|
||||
_, ids = reply_label_ids(tokenizer, probe, qtype.label_set)
|
||||
self.label_ids.update(zip(qtype.label_set, ids))
|
||||
|
||||
def limits(self) -> DecisionLimits:
|
||||
return DecisionLimits(max_questions=64, max_options=len(LABELS))
|
||||
@@ -173,7 +185,7 @@ class NextTokenStrategy(ReadStrategy):
|
||||
"these chat options end the prompt differently, so the "
|
||||
"labels' tokens are unknown"
|
||||
)
|
||||
slots.append(self.label_ids[: len(q.labels)])
|
||||
slots.append([self.label_ids[label] for label in q.labels])
|
||||
engine_inputs.append(engine_input)
|
||||
|
||||
label_reads = await next_token_label_reads(
|
||||
|
||||
Reference in New Issue
Block a user