[Feat] Structured decisions endpoint (/v1/systemone) (#59299)

Signed-off-by: Matt Mastracci <matthew@mastracci.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
Co-authored-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com>
Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
Matt Mastracci
2026-10-08 14:48:12 +00:00
committed by GitHub
co-authored by Claude Opus 5.5 Lucas Wilkinson Lucas Wilkinson Francisco Javier Arceo Chauncey
parent 273e099693
commit d33872dfeb
17 changed files with 1285 additions and 61 deletions
+33
View File
@@ -53,6 +53,12 @@ def register_generate_api_routers(app: FastAPI):
register_generative_scoring_api_router(app)
from .structured_decisions.api_router import (
register_structured_decisions_api_router,
)
register_structured_decisions_api_router(app)
async def init_generate_state(
engine_client: "EngineClient",
@@ -220,3 +226,30 @@ async def init_generate_state(
state.openai_serving_models,
request_logger=request_logger,
)
from .structured_decisions.serving import ServingStructuredDecisions
from .structured_decisions.strategies import ReadContext, select_read_strategy
strategy_cls = (
select_read_strategy(engine_client.model_config)
if "generate" in supported_tasks
and getattr(args, "enable_structured_decisions", False)
else None
)
state.serving_structured_decisions = (
ServingStructuredDecisions(
state.openai_serving_models,
strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
chat_template=resolved_chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
),
request_logger=request_logger,
)
if strategy_cls is not None
else None
)
@@ -11,13 +11,14 @@ logits (task="generate").
import asyncio
import math
import time
from collections.abc import AsyncGenerator, Mapping
from collections.abc import Mapping
from typing import Literal
from fastapi import Request
from pydantic import Field
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.generate.label_reads import next_token_label_reads
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import (
ErrorResponse,
@@ -28,7 +29,6 @@ from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.inputs import EngineInput, tokens_input
from vllm.logger import init_logger
from vllm.outputs import RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
from vllm.tracing import (
@@ -37,7 +37,6 @@ from vllm.tracing import (
log_tracing_disabled_warning,
)
from vllm.utils import random_uuid
from vllm.utils.async_utils import merge_async_iterators
logger = init_logger(__name__)
@@ -271,37 +270,28 @@ class ServingGenerativeScoring(BaseServing):
else await self._get_trace_headers(raw_request.headers)
)
# Schedule requests for all inputs
generators: list[AsyncGenerator[RequestOutput, None]] = []
for i, engine_input in enumerate(engine_inputs):
request_id_item = f"{request_id}-{i}"
self._log_inputs(
request_id_item,
f"{request_id}-{i}",
engine_input,
params=sampling_params,
lora_request=lora_request,
)
generator = self.engine_client.generate(
engine_input,
sampling_params,
request_id_item,
try:
reads = await next_token_label_reads(
self.engine_client,
engine_inputs,
[sampling_params] * len(engine_inputs),
request_id,
lora_request=lora_request,
trace_headers=trace_headers,
priority=request.priority,
)
generators.append(generator)
# Collect results
result_generator = merge_async_iterators(*generators)
results: list[RequestOutput | None] = [None] * len(engine_inputs)
try:
async for i, res in result_generator:
results[i] = res
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
except ValueError as e:
return self.create_error_response(e)
except Exception as e:
logger.exception("Error during generation")
return self.create_error_response(e)
@@ -311,45 +301,8 @@ class ServingGenerativeScoring(BaseServing):
total_prompt_tokens = 0
total_completion_tokens = 0
for i, result in enumerate(results):
if result is None:
return self.create_error_response(
f"Failed to generate result for item {i}"
)
# Check for errors
if result.outputs and result.outputs[0].finish_reason == "error":
return self.create_error_response(f"Generation error for item {i}")
# Get logprobs from the generated token
if not result.outputs or len(result.outputs) == 0:
return self.create_error_response(f"No output generated for item {i}")
output = result.outputs[0]
if output.logprobs is None or len(output.logprobs) == 0:
return self.create_error_response(
f"No logprobs available for item {i}. "
"This might indicate an issue with logprobs configuration."
)
# The logprobs dict maps token_id -> Logprob object
# For logprobs=-1, this contains all vocab tokens
logprobs_dict = output.logprobs[0]
# Extract logprobs for label tokens
label_logprobs: dict[int, float] = {}
missing_tokens = []
for token_id in request.label_token_ids:
if token_id in logprobs_dict:
label_logprobs[token_id] = logprobs_dict[token_id].logprob
else:
missing_tokens.append(token_id)
if missing_tokens:
return self.create_error_response(
f"Token IDs {missing_tokens} not found in logprobs for item {i}. "
"This might indicate the tokens are outside the model's vocabulary."
)
for i, read in enumerate(reads):
label_logprobs = dict(zip(request.label_token_ids, read.logprobs))
# Compute probabilities based on apply_softmax setting
token_probs = self._compute_probabilities(
@@ -370,7 +323,7 @@ class ServingGenerativeScoring(BaseServing):
# Update token counts
total_prompt_tokens += prompt_token_counts[i]
total_completion_tokens += len(output.token_ids)
total_completion_tokens += len(read.result.outputs[0].token_ids)
# Build response
model_name = self.models.model_name(lora_request)
+76
View File
@@ -0,0 +1,76 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""One-token reads that return the logprobs of chosen label tokens."""
from collections.abc import AsyncGenerator, Mapping, Sequence
from dataclasses import dataclass
from vllm.engine.protocol import EngineClient
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.outputs import RequestOutput
from vllm.sampling_params import SamplingParams
from vllm.utils.async_utils import merge_async_iterators
@dataclass(frozen=True)
class LabelRead:
result: RequestOutput
logprobs: list[float]
"""Full-vocabulary logprob of each label token, in ``logprob_token_ids``
order."""
async def next_token_label_reads(
engine_client: EngineClient,
engine_inputs: Sequence[EngineInput],
sampling_params: Sequence[SamplingParams],
request_id: str,
*,
lora_request: LoRARequest | None = None,
trace_headers: Mapping[str, str] | None = None,
priority: int = 0,
) -> list[LabelRead]:
"""Runs one read per input at once, as request ``{request_id}-{i}``. Each
read's params set ``max_tokens=1`` and the label tokens as
``logprob_token_ids``. Raises ValueError naming the item when a read has
no output or lacks a label's logprob."""
generators: list[AsyncGenerator[RequestOutput, None]] = [
engine_client.generate(
engine_input,
params,
f"{request_id}-{i}",
lora_request=lora_request,
trace_headers=trace_headers,
priority=priority,
)
for i, (engine_input, params) in enumerate(zip(engine_inputs, sampling_params))
]
results: list[RequestOutput | None] = [None] * len(generators)
async for i, res in merge_async_iterators(*generators):
results[i] = res
reads = []
for i, (result, params) in enumerate(zip(results, sampling_params)):
if result is None:
raise ValueError(f"Failed to generate result for item {i}")
if not result.outputs:
raise ValueError(f"No output generated for item {i}")
output = result.outputs[0]
if output.finish_reason == "error":
raise ValueError(f"Generation error for item {i}")
if not output.logprobs:
raise ValueError(
f"No logprobs available for item {i}. "
"This might indicate an issue with logprobs configuration."
)
logprobs = output.logprobs[0]
label_ids = params.logprob_token_ids or []
missing = [t for t in label_ids if t not in logprobs]
if missing:
raise ValueError(
f"Token IDs {missing} not found in logprobs for item {i}. "
"This might indicate the tokens are outside the model's vocabulary."
)
reads.append(LabelRead(result, [logprobs[t].logprob for t in label_ids]))
return reads
@@ -0,0 +1,2 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -0,0 +1,53 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import StructuredDecisionRequest
from .serving import ServingStructuredDecisions
router = APIRouter()
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
return getattr(request.app.state, "serving_structured_decisions", None)
@router.post(
"/v1/systemone",
dependencies=[Depends(validate_json_request)],
responses={
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
},
)
@with_cancellation
@load_aware_call
async def create_structured_decision(
request: StructuredDecisionRequest, raw_request: Request
):
handler = structured_decisions(raw_request)
if handler is None:
raise NotImplementedError(
"The model does not support the structured decision API"
)
result = await handler.create_decision(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_structured_decisions_api_router(app: FastAPI):
if getattr(app.state.args, "enable_structured_decisions", False):
app.include_router(router)
@@ -0,0 +1,88 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Request and response models of the structured decisions API (/v1/systemone)."""
import time
from typing import Any, Literal
from pydantic import Field
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
from vllm.utils import random_uuid
class QuestionSpec(OpenAIBaseModel):
type: str
instructions: Any = ""
criteria: Any = None
class StructuredDecisionRequest(OpenAIBaseModel):
model: str | None = None
state: Any = Field(..., description="What the questions are about: text or JSON.")
questions: dict[str, QuestionSpec] = Field(
..., description="Question id to question, asked in this order."
)
instructions: str | None = Field(
default=None, description="Context placed ahead of the questions."
)
chat_template_kwargs: dict[str, Any] | None = None
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
request_id: str = Field(default_factory=random_uuid)
class DecisionUsage(OpenAIBaseModel):
input_tokens: int
output_tokens: int
class QuestionDiagnostics(OpenAIBaseModel):
label_mass: float = Field(
description="Probability the model put on the labels, over the full "
"vocabulary. A low value means most of the probability went to tokens "
"that are not labels."
)
argmax_is_label: bool
class StructuredDecisionResponse(OpenAIBaseModel):
id: str
object: Literal["structured_decision"] = "structured_decision"
created: int = Field(default_factory=lambda: int(time.time()))
model: str
answers: dict[str, dict[str, Any]]
usage: DecisionUsage
diagnostics: dict[str, QuestionDiagnostics]
class ReadPromptRequest(OpenAIBaseModel):
"""Chat options for one read's prompt: the state and the question in the
user turn, ending at the generation prompt so the label is the reply's
first token. Thinking is off unless the request turns it on, or the label
would follow a thought rather than start the reply."""
chat_template_kwargs: dict[str, Any] | None = None
def build_chat_params(
self,
default_template: str | None,
default_template_content_format: ChatTemplateContentFormatOption,
) -> ChatParams:
return ChatParams(
chat_template=default_template,
chat_template_content_format=default_template_content_format,
chat_template_kwargs=merge_kwargs(
merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs),
dict(add_generation_prompt=True, continue_final_message=False),
),
)
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
return TokenizeParams(
max_total_tokens=model_config.max_model_len,
max_output_tokens=1,
add_special_tokens=False,
)
@@ -0,0 +1,160 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Question types for structured decisions.
A question type turns a request's criteria into options, writes the question
for the model with a label per option, and builds the answer from the label
probabilities.
"""
import math
import string
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: Options are labeled A to Z in order. A single capital letter is one token
#: at the start of a reply for the tokenizers tested; every read checks it.
LABELS = tuple(string.ascii_uppercase)
@dataclass(frozen=True)
class Option:
name: str
description: str | None = None
@dataclass(frozen=True)
class Question:
id: str
type: "QuestionType"
instructions: str
options: tuple[Option, ...]
labels: tuple[str, ...]
class QuestionType(ABC):
name: ClassVar[str]
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@abstractmethod
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
"""The answer for ``question``. ``probs[i]`` is the probability of
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(
f"{label}: {o.name} - {o.description}"
if o.description
else f"{label}: {o.name}"
)
lines.append("Answer with the letter of one option only.")
return "\n".join(lines)
QUESTION_TYPES: dict[str, QuestionType] = {}
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
if cls.name in QUESTION_TYPES:
raise ValueError(f"question type {cls.name!r} is already registered")
QUESTION_TYPES[cls.name] = cls()
return cls
def get_question_type(name: str) -> QuestionType:
try:
return QUESTION_TYPES[name]
except KeyError:
raise StructuredDecisionError(
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
) from None
def build_question(
qid: str,
type_name: str,
instructions: Any,
criteria: Any,
max_options: int,
) -> Question:
if not qid:
raise StructuredDecisionError("question ids must be non-empty")
qtype = get_question_type(type_name)
options = qtype.parse_options(qid, criteria)
if not options:
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(LABELS))
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
)
if not isinstance(instructions, str):
instructions = "" if instructions is None else str(instructions)
return Question(
id=qid,
type=qtype,
instructions=instructions,
options=tuple(options),
labels=LABELS[: len(options)],
)
def argmax(values: list[float]) -> int:
return max(range(len(values)), key=values.__getitem__)
def label_softmax(logprobs: list[float]) -> list[float]:
"""Softmax of the label logprobs."""
top = max(logprobs)
weights = [math.exp(lp - top) for lp in logprobs]
total = sum(weights)
return [w / total for w in weights]
@register_question_type
class ChoiceQuestion(QuestionType):
"""Pick one option. ``criteria`` maps each option name to a description,
or to null."""
name = "choice"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, dict) or not criteria:
raise StructuredDecisionError(
f"question {qid!r}: choice criteria must map option names to "
"a description or null"
)
return [
Option(str(name), None if desc is None else str(desc))
for name, desc in criteria.items()
]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
"choice": question.options[top].name,
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
# The chosen label's probability over the whole vocabulary, so a
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@@ -0,0 +1,121 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import json
from typing import Any
from fastapi import Request
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.logger import init_logger
from .protocol import (
DecisionUsage,
QuestionDiagnostics,
StructuredDecisionRequest,
StructuredDecisionResponse,
)
from .question_types import Question, StructuredDecisionError, build_question
from .strategies import DecisionLimits, ReadStrategy
logger = init_logger(__name__)
def state_text(state: Any) -> str:
return state if isinstance(state, str) else json.dumps(state)
def parse_questions(
request: StructuredDecisionRequest,
limits: DecisionLimits,
) -> list[Question]:
if not request.questions:
raise StructuredDecisionError("questions: needs at least one question")
if len(request.questions) > limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {limits.max_questions} for this model"
)
questions = []
for qid, spec in request.questions.items():
if spec.model_extra:
raise StructuredDecisionError(
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
)
questions.append(
build_question(
qid,
spec.type,
spec.instructions,
spec.criteria,
limits.max_options,
)
)
return questions
class ServingStructuredDecisions(BaseServing):
def __init__(
self,
models: OpenAIServingModels,
strategy: ReadStrategy,
*,
request_logger: RequestLogger | None = None,
) -> None:
model_config = strategy.context.engine_client.model_config
super().__init__(
models=models, model_config=model_config, request_logger=request_logger
)
self.strategy = strategy
self.limits = strategy.limits()
async def create_decision(
self,
request: StructuredDecisionRequest,
raw_request: Request | None = None,
) -> StructuredDecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine_client = self.strategy.context.engine_client
if engine_client.errored:
raise engine_client.dead_error
base_id = self._base_request_id(raw_request, default=request.request_id)
request_id = f"decision-{base_id}"
try:
questions = parse_questions(request, self.limits)
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine_client.check_admission(len(questions))
reads = await self.strategy.read(
questions,
request.instructions,
state_text(request.state),
request_id=request_id,
chat_template_kwargs=request.chat_template_kwargs,
lora_request=lora_request,
priority=request.priority,
)
except StructuredDecisionError as e:
return self.create_error_response(e)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
answers: dict[str, dict[str, Any]] = {}
diagnostics: dict[str, QuestionDiagnostics] = {}
for q, read in zip(questions, reads):
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
diagnostics[q.id] = QuestionDiagnostics(
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
)
return StructuredDecisionResponse(
id=request_id,
model=self.models.model_name(lora_request),
answers=answers,
usage=DecisionUsage(
input_tokens=sum(r.input_tokens for r in reads),
output_tokens=sum(r.output_tokens for r in reads),
),
diagnostics=diagnostics,
)
@@ -0,0 +1,235 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Read strategies: how a model is asked for label probabilities.
Only models in NEXT_TOKEN_ARCHITECTURES are currently supported.
"""
import math
from abc import ABC, abstractmethod
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any
from vllm.config import ModelConfig
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
from vllm.entrypoints.generate.label_reads import next_token_label_reads
from vllm.inputs import EngineInput
from vllm.lora.request import LoRARequest
from vllm.renderers.inputs.preprocess import extract_prompt_components
from vllm.renderers.online_renderer import OnlineRenderer
from vllm.sampling_params import SamplingParams
from vllm.tokenizers import TokenizerLike
from .protocol import ReadPromptRequest
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
@dataclass
class ReadContext:
engine_client: EngineClient
online_renderer: OnlineRenderer
chat_template: str | None
chat_template_content_format: ChatTemplateContentFormatOption
default_chat_template_kwargs: dict[str, Any]
@dataclass(frozen=True)
class DecisionLimits:
max_questions: int
max_options: int
@dataclass
class QuestionRead:
probs: list[float]
label_mass: float
argmax_is_label: bool
input_tokens: int
output_tokens: int
class ReadStrategy(ABC):
def __init__(self, context: ReadContext):
self.context = context
@abstractmethod
def limits(self) -> DecisionLimits: ...
@abstractmethod
async def read(
self,
questions: list[Question],
instructions: str | None,
state: str,
*,
request_id: str,
chat_template_kwargs: dict[str, Any] | None,
lora_request: LoRARequest | None,
priority: int,
) -> list[QuestionRead]:
"""One read per question, in order."""
def reply_label_ids(
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
) -> tuple[list[int], list[int]]:
"""The prompt's ids after its last added token, and the token each of LABELS
adds after them as the first token of the reply. Raises ValueError if a
label is not one distinct token there."""
added = set(tokenizer.get_added_vocab().values())
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
tail = list(prompt_ids[start:])
text = tokenizer.decode(tail)
ids: list[int] = []
for label in LABELS:
extended = tokenizer.encode(text + label, add_special_tokens=False)
if extended[:-1] != tail or extended[-1] in ids:
raise ValueError(
f"label {label!r} is not one distinct token after this model's "
"chat prompt"
)
ids.append(extended[-1])
return tail, ids
class NextTokenStrategy(ReadStrategy):
"""Autoregressive models. Each question is one request: the state, then the
question with its labeled options, and the logprobs of the label tokens as
the reply's first token. The requests share the state. With prefix caching,
it is prefilled once."""
def __init__(self, context: ReadContext):
super().__init__(context)
tokenizer = context.online_renderer.renderer.get_tokenizer()
probe = tokenizer.apply_chat_template(
[{"role": "user", "content": "x"}],
**{
**context.default_chat_template_kwargs,
"chat_template": context.chat_template,
"add_generation_prompt": True,
"tokenize": True,
"return_dict": False,
"enable_thinking": False,
},
)
if isinstance(probe, str):
probe = tokenizer.encode(probe, add_special_tokens=False)
# Every prompt ends with the same generation prompt, so the labels'
# tokens are the same for every question.
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
def limits(self) -> DecisionLimits:
return DecisionLimits(max_questions=64, max_options=len(LABELS))
def _read_request(
self, chat_template_kwargs: dict[str, Any] | None
) -> ReadPromptRequest:
if (chat_template_kwargs or {}).get("enable_thinking"):
raise StructuredDecisionError(
"a read is the reply's first token, so thinking must be off"
)
return ReadPromptRequest(chat_template_kwargs=chat_template_kwargs)
async def _render(
self, read_request: ReadPromptRequest, messages: list[dict[str, Any]]
) -> tuple[EngineInput, list[int]]:
ctx = self.context
_, (engine_input,) = await ctx.online_renderer.preprocess_chat(
read_request,
messages,
default_template=ctx.chat_template,
default_template_content_format=ctx.chat_template_content_format,
default_template_kwargs=ctx.default_chat_template_kwargs,
)
prompt_ids = extract_prompt_components(
ctx.engine_client.model_config, engine_input
).token_ids
return engine_input, list(prompt_ids or [])
async def read(
self,
questions: list[Question],
instructions: str | None,
state: str,
*,
request_id: str,
chat_template_kwargs: dict[str, Any] | None,
lora_request: LoRARequest | None,
priority: int,
) -> list[QuestionRead]:
ctx = self.context
read_request = self._read_request(chat_template_kwargs)
slots, engine_inputs = [], []
for q in questions:
messages = [{"role": "user", "content": f"{state}\n\n{q.type.prompt(q)}"}]
if instructions:
messages.insert(0, {"role": "system", "content": instructions})
engine_input, prompt_ids = await self._render(read_request, messages)
if prompt_ids[len(prompt_ids) - len(self.tail) :] != self.tail:
raise StructuredDecisionError(
"these chat options end the prompt differently, so the "
"labels' tokens are unknown"
)
slots.append(self.label_ids[: len(q.labels)])
engine_inputs.append(engine_input)
label_reads = await next_token_label_reads(
ctx.engine_client,
engine_inputs,
[
SamplingParams(max_tokens=1, temperature=0.0, logprob_token_ids=ids)
for ids in slots
],
request_id,
lora_request=lora_request,
priority=priority,
)
reads = []
for ids, label_read in zip(slots, label_reads):
output = label_read.result.outputs[0]
reads.append(
QuestionRead(
probs=label_softmax(label_read.logprobs),
label_mass=sum(math.exp(lp) for lp in label_read.logprobs),
argmax_is_label=bool(output.token_ids)
and output.token_ids[0] in ids,
input_tokens=len(label_read.result.prompt_token_ids or ()),
output_tokens=len(output.token_ids),
)
)
return reads
NEXT_TOKEN_ARCHITECTURES = frozenset(
{
"Qwen3ForCausalLM",
"Qwen3_5ForConditionalGeneration",
"Qwen3_5MoeForConditionalGeneration",
}
)
# label_mass sums the labels' full-vocabulary probabilities. The logprobs modes
# return those probabilities. The logits modes return raw logits instead.
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
"""Raises ValueError when the model cannot serve structured decisions. The
server opted in with --enable-structured-decisions, so startup fails."""
if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES:
raise ValueError(
"--enable-structured-decisions does not support "
f"{model_config.architecture}. Supported architectures: "
f"{sorted(NEXT_TOKEN_ARCHITECTURES)}"
)
if model_config.logprobs_mode not in LOGPROBS_MODES:
raise ValueError(
"--enable-structured-decisions needs --logprobs-mode raw_logprobs or "
f"processed_logprobs, not {model_config.logprobs_mode}"
)
return NextTokenStrategy
+2
View File
@@ -163,6 +163,8 @@ class BaseFrontendArgs:
enable_tokenizer_info_endpoint: bool = False
"""Enable the `/tokenizer_info` endpoint. May expose chat
templates and other tokenizer configuration."""
enable_structured_decisions: bool = False
"""Enable the `/v1/systemone` structured decisions endpoint."""
enable_log_outputs: bool = False
"""If set to True, log model outputs (generations). Requires
`--enable-log-requests`. Output text and finish reasons are logged at INFO,