[Frontend] structured decisions: noul and score question types (#60696)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Co-authored-by: DeepSeek <noreply@deepseek.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Misha Goin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
DeepSeek
Claude Opus 5.5
Misha Goin
parent
88a419649e
commit
8cbd5d0300
@@ -18,8 +18,8 @@ class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: Options are labeled A to Z in order. A single capital letter is one token
|
||||
#: at the start of a reply for the tokenizers tested; every read checks it.
|
||||
#: A choice labels its options A to Z in order. A single capital letter is one
|
||||
#: token at the start of a reply for the tokenizers tested. Every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@@ -40,6 +40,9 @@ class Question:
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
#: The labels of a question's options, in option order.
|
||||
label_set: ClassVar[tuple[str, ...]] = LABELS
|
||||
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
@@ -52,16 +55,22 @@ class QuestionType(ABC):
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str | None:
|
||||
"""The option's line in the prompt, or None to leave it out."""
|
||||
return (
|
||||
f"{label}: {option.name} - {option.description}"
|
||||
if option.description
|
||||
else f"{label}: {option.name}"
|
||||
)
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(
|
||||
f"{label}: {o.name} - {o.description}"
|
||||
if o.description
|
||||
else f"{label}: {o.name}"
|
||||
)
|
||||
lines.append("Answer with the letter of one option only.")
|
||||
line = self.reply_line(label, o)
|
||||
if line is not None:
|
||||
lines.append(line)
|
||||
lines.append(self.reply_instruction)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
@@ -100,7 +109,7 @@ def build_question(
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(LABELS))
|
||||
limit = min(max_options, len(qtype.label_set))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
@@ -112,7 +121,7 @@ def build_question(
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
labels=qtype.label_set[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
@@ -158,3 +167,74 @@ class ChoiceQuestion(QuestionType):
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class NoulQuestion(QuestionType):
|
||||
"""Yes or no. ``criteria`` may describe what true and false mean."""
|
||||
|
||||
name = "noul"
|
||||
label_set = ("yes", "no")
|
||||
reply_instruction = "Answer with yes or no only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if criteria is not None and (
|
||||
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
|
||||
):
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: noul criteria must be an object with true and false"
|
||||
)
|
||||
criteria = criteria or {}
|
||||
true = criteria.get("true")
|
||||
false = criteria.get("false")
|
||||
return [
|
||||
Option("yes", None if true is None else str(true)),
|
||||
Option("no", None if false is None else str(false)),
|
||||
]
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str | None:
|
||||
return f"{label}: {option.description}" if option.description else None
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.name,
|
||||
"noul": probs[0],
|
||||
"probabilities": {"yes": probs[0], "no": probs[1]},
|
||||
# The chosen label's probability over the whole vocabulary, like a
|
||||
# choice's confidence.
|
||||
"confidence": max(probs) * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ScoreQuestion(QuestionType):
|
||||
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
|
||||
level names, from the first to the last level of the scale."""
|
||||
|
||||
name = "score"
|
||||
# 0-indexed like the answer, so a level's label is its score.
|
||||
label_set = tuple(string.digits)
|
||||
reply_instruction = "Answer with the number of one level only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, list) or len(criteria) < 2:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: score criteria must be an ordered list of levels"
|
||||
)
|
||||
return [Option(str(level)) for level in criteria]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
# The expected level, 0-indexed, so a level's index in the request's
|
||||
# criteria is its value.
|
||||
"score": sum(i * p for i, p in enumerate(probs)),
|
||||
"legend": {str(i): o.name for i, o in enumerate(question.options)},
|
||||
"probabilities": {str(i): p for i, p in enumerate(probs)},
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
@@ -23,7 +23,13 @@ from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .protocol import ReadPromptRequest
|
||||
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
QUESTION_TYPES,
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
label_softmax,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -73,17 +79,19 @@ class ReadStrategy(ABC):
|
||||
|
||||
|
||||
def reply_label_ids(
|
||||
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
|
||||
tokenizer: TokenizerLike,
|
||||
prompt_ids: Sequence[int],
|
||||
labels: Sequence[str] = LABELS,
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""The prompt's ids after its last added token, and the token each of LABELS
|
||||
adds after them as the first token of the reply. Raises ValueError if a
|
||||
label is not one distinct token there."""
|
||||
"""The prompt's ids after its last added token, and the token each of
|
||||
``labels`` adds after them as the first token of the reply. Raises
|
||||
ValueError if a label is not one distinct token there."""
|
||||
added = set(tokenizer.get_added_vocab().values())
|
||||
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
|
||||
tail = list(prompt_ids[start:])
|
||||
text = tokenizer.decode(tail)
|
||||
ids: list[int] = []
|
||||
for label in LABELS:
|
||||
for label in labels:
|
||||
extended = tokenizer.encode(text + label, add_special_tokens=False)
|
||||
if extended[:-1] != tail or extended[-1] in ids:
|
||||
raise ValueError(
|
||||
@@ -116,9 +124,13 @@ class NextTokenStrategy(ReadStrategy):
|
||||
)
|
||||
if isinstance(probe, str):
|
||||
probe = tokenizer.encode(probe, add_special_tokens=False)
|
||||
# Every prompt ends with the same generation prompt, so the labels'
|
||||
# tokens are the same for every question.
|
||||
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
|
||||
# Every prompt ends with the same generation prompt, so a label's token
|
||||
# is the same for every question.
|
||||
self.tail, _ = reply_label_ids(tokenizer, probe, ())
|
||||
self.label_ids: dict[str, int] = {}
|
||||
for qtype in QUESTION_TYPES.values():
|
||||
_, ids = reply_label_ids(tokenizer, probe, qtype.label_set)
|
||||
self.label_ids.update(zip(qtype.label_set, ids))
|
||||
|
||||
def limits(self) -> DecisionLimits:
|
||||
return DecisionLimits(max_questions=64, max_options=len(LABELS))
|
||||
@@ -173,7 +185,7 @@ class NextTokenStrategy(ReadStrategy):
|
||||
"these chat options end the prompt differently, so the "
|
||||
"labels' tokens are unknown"
|
||||
)
|
||||
slots.append(self.label_ids[: len(q.labels)])
|
||||
slots.append([self.label_ids[label] for label in q.labels])
|
||||
engine_inputs.append(engine_input)
|
||||
|
||||
label_reads = await next_token_label_reads(
|
||||
|
||||
Reference in New Issue
Block a user