[Frontend] structured decisions: noul and score question types (#60696)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Co-authored-by: DeepSeek <noreply@deepseek.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
Matt Mastracci
2026-10-09 04:48:16 +00:00
committed by GitHub
co-authored by DeepSeek Claude Opus 5.5 Misha Goin
parent 88a419649e
commit 8cbd5d0300
6 changed files with 270 additions and 27 deletions
@@ -18,8 +18,8 @@ class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: Options are labeled A to Z in order. A single capital letter is one token
#: at the start of a reply for the tokenizers tested; every read checks it.
#: A choice labels its options A to Z in order. A single capital letter is one
#: token at the start of a reply for the tokenizers tested. Every read checks it.
LABELS = tuple(string.ascii_uppercase)
@@ -40,6 +40,9 @@ class Question:
class QuestionType(ABC):
name: ClassVar[str]
#: The labels of a question's options, in option order.
label_set: ClassVar[tuple[str, ...]] = LABELS
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@@ -52,16 +55,22 @@ class QuestionType(ABC):
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def reply_line(self, label: str, option: Option) -> str | None:
"""The option's line in the prompt, or None to leave it out."""
return (
f"{label}: {option.name} - {option.description}"
if option.description
else f"{label}: {option.name}"
)
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(
f"{label}: {o.name} - {o.description}"
if o.description
else f"{label}: {o.name}"
)
lines.append("Answer with the letter of one option only.")
line = self.reply_line(label, o)
if line is not None:
lines.append(line)
lines.append(self.reply_instruction)
return "\n".join(lines)
@@ -100,7 +109,7 @@ def build_question(
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(LABELS))
limit = min(max_options, len(qtype.label_set))
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
@@ -112,7 +121,7 @@ def build_question(
type=qtype,
instructions=instructions,
options=tuple(options),
labels=LABELS[: len(options)],
labels=qtype.label_set[: len(options)],
)
@@ -158,3 +167,74 @@ class ChoiceQuestion(QuestionType):
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@register_question_type
class NoulQuestion(QuestionType):
"""Yes or no. ``criteria`` may describe what true and false mean."""
name = "noul"
label_set = ("yes", "no")
reply_instruction = "Answer with yes or no only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if criteria is not None and (
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
):
raise StructuredDecisionError(
f"question {qid!r}: noul criteria must be an object with true and false"
)
criteria = criteria or {}
true = criteria.get("true")
false = criteria.get("false")
return [
Option("yes", None if true is None else str(true)),
Option("no", None if false is None else str(false)),
]
def reply_line(self, label: str, option: Option) -> str | None:
return f"{label}: {option.description}" if option.description else None
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {
"type": self.name,
"noul": probs[0],
"probabilities": {"yes": probs[0], "no": probs[1]},
# The chosen label's probability over the whole vocabulary, like a
# choice's confidence.
"confidence": max(probs) * label_mass,
}
@register_question_type
class ScoreQuestion(QuestionType):
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
level names, from the first to the last level of the scale."""
name = "score"
# 0-indexed like the answer, so a level's label is its score.
label_set = tuple(string.digits)
reply_instruction = "Answer with the number of one level only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, list) or len(criteria) < 2:
raise StructuredDecisionError(
f"question {qid!r}: score criteria must be an ordered list of levels"
)
return [Option(str(level)) for level in criteria]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
# The expected level, 0-indexed, so a level's index in the request's
# criteria is its value.
"score": sum(i * p for i, p in enumerate(probs)),
"legend": {str(i): o.name for i, o in enumerate(question.options)},
"probabilities": {str(i): p for i, p in enumerate(probs)},
"confidence": probs[top] * label_mass,
}
@@ -23,7 +23,13 @@ from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
from .protocol import ReadPromptRequest
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
from .question_types import (
LABELS,
QUESTION_TYPES,
Question,
StructuredDecisionError,
label_softmax,
)
@dataclass
@@ -73,17 +79,19 @@ class ReadStrategy(ABC):
def reply_label_ids(
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
tokenizer: TokenizerLike,
prompt_ids: Sequence[int],
labels: Sequence[str] = LABELS,
) -> tuple[list[int], list[int]]:
"""The prompt's ids after its last added token, and the token each of LABELS
adds after them as the first token of the reply. Raises ValueError if a
label is not one distinct token there."""
"""The prompt's ids after its last added token, and the token each of
``labels`` adds after them as the first token of the reply. Raises
ValueError if a label is not one distinct token there."""
added = set(tokenizer.get_added_vocab().values())
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
tail = list(prompt_ids[start:])
text = tokenizer.decode(tail)
ids: list[int] = []
for label in LABELS:
for label in labels:
extended = tokenizer.encode(text + label, add_special_tokens=False)
if extended[:-1] != tail or extended[-1] in ids:
raise ValueError(
@@ -116,9 +124,13 @@ class NextTokenStrategy(ReadStrategy):
)
if isinstance(probe, str):
probe = tokenizer.encode(probe, add_special_tokens=False)
# Every prompt ends with the same generation prompt, so the labels'
# tokens are the same for every question.
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
# Every prompt ends with the same generation prompt, so a label's token
# is the same for every question.
self.tail, _ = reply_label_ids(tokenizer, probe, ())
self.label_ids: dict[str, int] = {}
for qtype in QUESTION_TYPES.values():
_, ids = reply_label_ids(tokenizer, probe, qtype.label_set)
self.label_ids.update(zip(qtype.label_set, ids))
def limits(self) -> DecisionLimits:
return DecisionLimits(max_questions=64, max_options=len(LABELS))
@@ -173,7 +185,7 @@ class NextTokenStrategy(ReadStrategy):
"these chat options end the prompt differently, so the "
"labels' tokens are unknown"
)
slots.append(self.label_ids[: len(q.labels)])
slots.append([self.label_ids[label] for label in q.labels])
engine_inputs.append(engine_input)
label_reads = await next_token_label_reads(