[Feat] Structured decisions endpoint (/v1/systemone) (#59299)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
Lucas Wilkinson
Francisco Javier Arceo
Chauncey
parent
273e099693
commit
d33872dfeb
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
|
||||
return getattr(request.app.state, "serving_structured_decisions", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/systemone",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_structured_decision(
|
||||
request: StructuredDecisionRequest, raw_request: Request
|
||||
):
|
||||
handler = structured_decisions(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support the structured decision API"
|
||||
)
|
||||
result = await handler.create_decision(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
if getattr(app.state.args, "enable_structured_decisions", False):
|
||||
app.include_router(router)
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Request and response models of the structured decisions API (/v1/systemone)."""
|
||||
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
class QuestionSpec(OpenAIBaseModel):
|
||||
type: str
|
||||
instructions: Any = ""
|
||||
criteria: Any = None
|
||||
|
||||
|
||||
class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
model: str | None = None
|
||||
state: Any = Field(..., description="What the questions are about: text or JSON.")
|
||||
questions: dict[str, QuestionSpec] = Field(
|
||||
..., description="Question id to question, asked in this order."
|
||||
)
|
||||
instructions: str | None = Field(
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
request_id: str = Field(default_factory=random_uuid)
|
||||
|
||||
|
||||
class DecisionUsage(OpenAIBaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class QuestionDiagnostics(OpenAIBaseModel):
|
||||
label_mass: float = Field(
|
||||
description="Probability the model put on the labels, over the full "
|
||||
"vocabulary. A low value means most of the probability went to tokens "
|
||||
"that are not labels."
|
||||
)
|
||||
argmax_is_label: bool
|
||||
|
||||
|
||||
class StructuredDecisionResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
object: Literal["structured_decision"] = "structured_decision"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
answers: dict[str, dict[str, Any]]
|
||||
usage: DecisionUsage
|
||||
diagnostics: dict[str, QuestionDiagnostics]
|
||||
|
||||
|
||||
class ReadPromptRequest(OpenAIBaseModel):
|
||||
"""Chat options for one read's prompt: the state and the question in the
|
||||
user turn, ending at the generation prompt so the label is the reply's
|
||||
first token. Thinking is off unless the request turns it on, or the label
|
||||
would follow a thought rather than start the reply."""
|
||||
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> ChatParams:
|
||||
return ChatParams(
|
||||
chat_template=default_template,
|
||||
chat_template_content_format=default_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs),
|
||||
dict(add_generation_prompt=True, continue_final_message=False),
|
||||
),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=1,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Question types for structured decisions.
|
||||
|
||||
A question type turns a request's criteria into options, writes the question
|
||||
for the model with a label per option, and builds the answer from the label
|
||||
probabilities.
|
||||
"""
|
||||
|
||||
import math
|
||||
import string
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: Options are labeled A to Z in order. A single capital letter is one token
|
||||
#: at the start of a reply for the tokenizers tested; every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Option:
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Question:
|
||||
id: str
|
||||
type: "QuestionType"
|
||||
instructions: str
|
||||
options: tuple[Option, ...]
|
||||
labels: tuple[str, ...]
|
||||
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
|
||||
@abstractmethod
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
"""The answer for ``question``. ``probs[i]`` is the probability of
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(
|
||||
f"{label}: {o.name} - {o.description}"
|
||||
if o.description
|
||||
else f"{label}: {o.name}"
|
||||
)
|
||||
lines.append("Answer with the letter of one option only.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
QUESTION_TYPES: dict[str, QuestionType] = {}
|
||||
|
||||
|
||||
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
|
||||
if cls.name in QUESTION_TYPES:
|
||||
raise ValueError(f"question type {cls.name!r} is already registered")
|
||||
QUESTION_TYPES[cls.name] = cls()
|
||||
return cls
|
||||
|
||||
|
||||
def get_question_type(name: str) -> QuestionType:
|
||||
try:
|
||||
return QUESTION_TYPES[name]
|
||||
except KeyError:
|
||||
raise StructuredDecisionError(
|
||||
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
|
||||
) from None
|
||||
|
||||
|
||||
def build_question(
|
||||
qid: str,
|
||||
type_name: str,
|
||||
instructions: Any,
|
||||
criteria: Any,
|
||||
max_options: int,
|
||||
) -> Question:
|
||||
if not qid:
|
||||
raise StructuredDecisionError("question ids must be non-empty")
|
||||
qtype = get_question_type(type_name)
|
||||
options = qtype.parse_options(qid, criteria)
|
||||
if not options:
|
||||
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(LABELS))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
)
|
||||
if not isinstance(instructions, str):
|
||||
instructions = "" if instructions is None else str(instructions)
|
||||
return Question(
|
||||
id=qid,
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def argmax(values: list[float]) -> int:
|
||||
return max(range(len(values)), key=values.__getitem__)
|
||||
|
||||
|
||||
def label_softmax(logprobs: list[float]) -> list[float]:
|
||||
"""Softmax of the label logprobs."""
|
||||
top = max(logprobs)
|
||||
weights = [math.exp(lp - top) for lp in logprobs]
|
||||
total = sum(weights)
|
||||
return [w / total for w in weights]
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ChoiceQuestion(QuestionType):
|
||||
"""Pick one option. ``criteria`` maps each option name to a description,
|
||||
or to null."""
|
||||
|
||||
name = "choice"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, dict) or not criteria:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: choice criteria must map option names to "
|
||||
"a description or null"
|
||||
)
|
||||
return [
|
||||
Option(str(name), None if desc is None else str(desc))
|
||||
for name, desc in criteria.items()
|
||||
]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
"choice": question.options[top].name,
|
||||
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
|
||||
# The chosen label's probability over the whole vocabulary, so a
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
QuestionDiagnostics,
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
from .question_types import Question, StructuredDecisionError, build_question
|
||||
from .strategies import DecisionLimits, ReadStrategy
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def state_text(state: Any) -> str:
|
||||
return state if isinstance(state, str) else json.dumps(state)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
request: StructuredDecisionRequest,
|
||||
limits: DecisionLimits,
|
||||
) -> list[Question]:
|
||||
if not request.questions:
|
||||
raise StructuredDecisionError("questions: needs at least one question")
|
||||
if len(request.questions) > limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {limits.max_questions} for this model"
|
||||
)
|
||||
questions = []
|
||||
for qid, spec in request.questions.items():
|
||||
if spec.model_extra:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
|
||||
)
|
||||
questions.append(
|
||||
build_question(
|
||||
qid,
|
||||
spec.type,
|
||||
spec.instructions,
|
||||
spec.criteria,
|
||||
limits.max_options,
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
class ServingStructuredDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
model_config = strategy.context.engine_client.model_config
|
||||
super().__init__(
|
||||
models=models, model_config=model_config, request_logger=request_logger
|
||||
)
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> StructuredDecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine_client = self.strategy.context.engine_client
|
||||
if engine_client.errored:
|
||||
raise engine_client.dead_error
|
||||
|
||||
base_id = self._base_request_id(raw_request, default=request.request_id)
|
||||
request_id = f"decision-{base_id}"
|
||||
try:
|
||||
questions = parse_questions(request, self.limits)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(len(questions))
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
state_text(request.state),
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
answers: dict[str, dict[str, Any]] = {}
|
||||
diagnostics: dict[str, QuestionDiagnostics] = {}
|
||||
for q, read in zip(questions, reads):
|
||||
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
|
||||
diagnostics[q.id] = QuestionDiagnostics(
|
||||
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
|
||||
)
|
||||
return StructuredDecisionResponse(
|
||||
id=request_id,
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=answers,
|
||||
usage=DecisionUsage(
|
||||
input_tokens=sum(r.input_tokens for r in reads),
|
||||
output_tokens=sum(r.output_tokens for r in reads),
|
||||
),
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
@@ -0,0 +1,235 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Read strategies: how a model is asked for label probabilities.
|
||||
|
||||
Only models in NEXT_TOKEN_ARCHITECTURES are currently supported.
|
||||
"""
|
||||
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.label_reads import next_token_label_reads
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .protocol import ReadPromptRequest
|
||||
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReadContext:
|
||||
engine_client: EngineClient
|
||||
online_renderer: OnlineRenderer
|
||||
chat_template: str | None
|
||||
chat_template_content_format: ChatTemplateContentFormatOption
|
||||
default_chat_template_kwargs: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecisionLimits:
|
||||
max_questions: int
|
||||
max_options: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuestionRead:
|
||||
probs: list[float]
|
||||
label_mass: float
|
||||
argmax_is_label: bool
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class ReadStrategy(ABC):
|
||||
def __init__(self, context: ReadContext):
|
||||
self.context = context
|
||||
|
||||
@abstractmethod
|
||||
def limits(self) -> DecisionLimits: ...
|
||||
|
||||
@abstractmethod
|
||||
async def read(
|
||||
self,
|
||||
questions: list[Question],
|
||||
instructions: str | None,
|
||||
state: str,
|
||||
*,
|
||||
request_id: str,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
) -> list[QuestionRead]:
|
||||
"""One read per question, in order."""
|
||||
|
||||
|
||||
def reply_label_ids(
|
||||
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""The prompt's ids after its last added token, and the token each of LABELS
|
||||
adds after them as the first token of the reply. Raises ValueError if a
|
||||
label is not one distinct token there."""
|
||||
added = set(tokenizer.get_added_vocab().values())
|
||||
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
|
||||
tail = list(prompt_ids[start:])
|
||||
text = tokenizer.decode(tail)
|
||||
ids: list[int] = []
|
||||
for label in LABELS:
|
||||
extended = tokenizer.encode(text + label, add_special_tokens=False)
|
||||
if extended[:-1] != tail or extended[-1] in ids:
|
||||
raise ValueError(
|
||||
f"label {label!r} is not one distinct token after this model's "
|
||||
"chat prompt"
|
||||
)
|
||||
ids.append(extended[-1])
|
||||
return tail, ids
|
||||
|
||||
|
||||
class NextTokenStrategy(ReadStrategy):
|
||||
"""Autoregressive models. Each question is one request: the state, then the
|
||||
question with its labeled options, and the logprobs of the label tokens as
|
||||
the reply's first token. The requests share the state. With prefix caching,
|
||||
it is prefilled once."""
|
||||
|
||||
def __init__(self, context: ReadContext):
|
||||
super().__init__(context)
|
||||
tokenizer = context.online_renderer.renderer.get_tokenizer()
|
||||
probe = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "x"}],
|
||||
**{
|
||||
**context.default_chat_template_kwargs,
|
||||
"chat_template": context.chat_template,
|
||||
"add_generation_prompt": True,
|
||||
"tokenize": True,
|
||||
"return_dict": False,
|
||||
"enable_thinking": False,
|
||||
},
|
||||
)
|
||||
if isinstance(probe, str):
|
||||
probe = tokenizer.encode(probe, add_special_tokens=False)
|
||||
# Every prompt ends with the same generation prompt, so the labels'
|
||||
# tokens are the same for every question.
|
||||
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
|
||||
|
||||
def limits(self) -> DecisionLimits:
|
||||
return DecisionLimits(max_questions=64, max_options=len(LABELS))
|
||||
|
||||
def _read_request(
|
||||
self, chat_template_kwargs: dict[str, Any] | None
|
||||
) -> ReadPromptRequest:
|
||||
if (chat_template_kwargs or {}).get("enable_thinking"):
|
||||
raise StructuredDecisionError(
|
||||
"a read is the reply's first token, so thinking must be off"
|
||||
)
|
||||
return ReadPromptRequest(chat_template_kwargs=chat_template_kwargs)
|
||||
|
||||
async def _render(
|
||||
self, read_request: ReadPromptRequest, messages: list[dict[str, Any]]
|
||||
) -> tuple[EngineInput, list[int]]:
|
||||
ctx = self.context
|
||||
_, (engine_input,) = await ctx.online_renderer.preprocess_chat(
|
||||
read_request,
|
||||
messages,
|
||||
default_template=ctx.chat_template,
|
||||
default_template_content_format=ctx.chat_template_content_format,
|
||||
default_template_kwargs=ctx.default_chat_template_kwargs,
|
||||
)
|
||||
prompt_ids = extract_prompt_components(
|
||||
ctx.engine_client.model_config, engine_input
|
||||
).token_ids
|
||||
return engine_input, list(prompt_ids or [])
|
||||
|
||||
async def read(
|
||||
self,
|
||||
questions: list[Question],
|
||||
instructions: str | None,
|
||||
state: str,
|
||||
*,
|
||||
request_id: str,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
) -> list[QuestionRead]:
|
||||
ctx = self.context
|
||||
read_request = self._read_request(chat_template_kwargs)
|
||||
|
||||
slots, engine_inputs = [], []
|
||||
for q in questions:
|
||||
messages = [{"role": "user", "content": f"{state}\n\n{q.type.prompt(q)}"}]
|
||||
if instructions:
|
||||
messages.insert(0, {"role": "system", "content": instructions})
|
||||
engine_input, prompt_ids = await self._render(read_request, messages)
|
||||
if prompt_ids[len(prompt_ids) - len(self.tail) :] != self.tail:
|
||||
raise StructuredDecisionError(
|
||||
"these chat options end the prompt differently, so the "
|
||||
"labels' tokens are unknown"
|
||||
)
|
||||
slots.append(self.label_ids[: len(q.labels)])
|
||||
engine_inputs.append(engine_input)
|
||||
|
||||
label_reads = await next_token_label_reads(
|
||||
ctx.engine_client,
|
||||
engine_inputs,
|
||||
[
|
||||
SamplingParams(max_tokens=1, temperature=0.0, logprob_token_ids=ids)
|
||||
for ids in slots
|
||||
],
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
reads = []
|
||||
for ids, label_read in zip(slots, label_reads):
|
||||
output = label_read.result.outputs[0]
|
||||
reads.append(
|
||||
QuestionRead(
|
||||
probs=label_softmax(label_read.logprobs),
|
||||
label_mass=sum(math.exp(lp) for lp in label_read.logprobs),
|
||||
argmax_is_label=bool(output.token_ids)
|
||||
and output.token_ids[0] in ids,
|
||||
input_tokens=len(label_read.result.prompt_token_ids or ()),
|
||||
output_tokens=len(output.token_ids),
|
||||
)
|
||||
)
|
||||
return reads
|
||||
|
||||
|
||||
NEXT_TOKEN_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"Qwen3ForCausalLM",
|
||||
"Qwen3_5ForConditionalGeneration",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# label_mass sums the labels' full-vocabulary probabilities. The logprobs modes
|
||||
# return those probabilities. The logits modes return raw logits instead.
|
||||
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
|
||||
|
||||
|
||||
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
|
||||
"""Raises ValueError when the model cannot serve structured decisions. The
|
||||
server opted in with --enable-structured-decisions, so startup fails."""
|
||||
if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES:
|
||||
raise ValueError(
|
||||
"--enable-structured-decisions does not support "
|
||||
f"{model_config.architecture}. Supported architectures: "
|
||||
f"{sorted(NEXT_TOKEN_ARCHITECTURES)}"
|
||||
)
|
||||
if model_config.logprobs_mode not in LOGPROBS_MODES:
|
||||
raise ValueError(
|
||||
"--enable-structured-decisions needs --logprobs-mode raw_logprobs or "
|
||||
f"processed_logprobs, not {model_config.logprobs_mode}"
|
||||
)
|
||||
return NextTokenStrategy
|
||||
Reference in New Issue
Block a user