[Feat] Structured decisions endpoint (/v1/systemone) (#59299)
Signed-off-by: Matt Mastracci <matthew@mastracci.com> Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com> Signed-off-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com> Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com> Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com> Co-authored-by: Francisco Javier Arceo <4163062+franciscojavierarceo@users.noreply.github.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
Lucas Wilkinson
Lucas Wilkinson
Francisco Javier Arceo
Chauncey
parent
273e099693
commit
d33872dfeb
@@ -53,6 +53,12 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_generative_scoring_api_router(app)
|
||||
|
||||
from .structured_decisions.api_router import (
|
||||
register_structured_decisions_api_router,
|
||||
)
|
||||
|
||||
register_structured_decisions_api_router(app)
|
||||
|
||||
|
||||
async def init_generate_state(
|
||||
engine_client: "EngineClient",
|
||||
@@ -220,3 +226,30 @@ async def init_generate_state(
|
||||
state.openai_serving_models,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
|
||||
from .structured_decisions.serving import ServingStructuredDecisions
|
||||
from .structured_decisions.strategies import ReadContext, select_read_strategy
|
||||
|
||||
strategy_cls = (
|
||||
select_read_strategy(engine_client.model_config)
|
||||
if "generate" in supported_tasks
|
||||
and getattr(args, "enable_structured_decisions", False)
|
||||
else None
|
||||
)
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
),
|
||||
request_logger=request_logger,
|
||||
)
|
||||
if strategy_cls is not None
|
||||
else None
|
||||
)
|
||||
|
||||
@@ -11,13 +11,14 @@ logits (task="generate").
|
||||
import asyncio
|
||||
import math
|
||||
import time
|
||||
from collections.abc import AsyncGenerator, Mapping
|
||||
from collections.abc import Mapping
|
||||
from typing import Literal
|
||||
|
||||
from fastapi import Request
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.generate.label_reads import next_token_label_reads
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import (
|
||||
ErrorResponse,
|
||||
@@ -28,7 +29,6 @@ from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.inputs import EngineInput, tokens_input
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
from vllm.tracing import (
|
||||
@@ -37,7 +37,6 @@ from vllm.tracing import (
|
||||
log_tracing_disabled_warning,
|
||||
)
|
||||
from vllm.utils import random_uuid
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
@@ -271,37 +270,28 @@ class ServingGenerativeScoring(BaseServing):
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
|
||||
# Schedule requests for all inputs
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = []
|
||||
for i, engine_input in enumerate(engine_inputs):
|
||||
request_id_item = f"{request_id}-{i}"
|
||||
|
||||
self._log_inputs(
|
||||
request_id_item,
|
||||
f"{request_id}-{i}",
|
||||
engine_input,
|
||||
params=sampling_params,
|
||||
lora_request=lora_request,
|
||||
)
|
||||
|
||||
generator = self.engine_client.generate(
|
||||
engine_input,
|
||||
sampling_params,
|
||||
request_id_item,
|
||||
try:
|
||||
reads = await next_token_label_reads(
|
||||
self.engine_client,
|
||||
engine_inputs,
|
||||
[sampling_params] * len(engine_inputs),
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=request.priority,
|
||||
)
|
||||
generators.append(generator)
|
||||
|
||||
# Collect results
|
||||
result_generator = merge_async_iterators(*generators)
|
||||
results: list[RequestOutput | None] = [None] * len(engine_inputs)
|
||||
|
||||
try:
|
||||
async for i, res in result_generator:
|
||||
results[i] = res
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
except ValueError as e:
|
||||
return self.create_error_response(e)
|
||||
except Exception as e:
|
||||
logger.exception("Error during generation")
|
||||
return self.create_error_response(e)
|
||||
@@ -311,45 +301,8 @@ class ServingGenerativeScoring(BaseServing):
|
||||
total_prompt_tokens = 0
|
||||
total_completion_tokens = 0
|
||||
|
||||
for i, result in enumerate(results):
|
||||
if result is None:
|
||||
return self.create_error_response(
|
||||
f"Failed to generate result for item {i}"
|
||||
)
|
||||
|
||||
# Check for errors
|
||||
if result.outputs and result.outputs[0].finish_reason == "error":
|
||||
return self.create_error_response(f"Generation error for item {i}")
|
||||
|
||||
# Get logprobs from the generated token
|
||||
if not result.outputs or len(result.outputs) == 0:
|
||||
return self.create_error_response(f"No output generated for item {i}")
|
||||
|
||||
output = result.outputs[0]
|
||||
if output.logprobs is None or len(output.logprobs) == 0:
|
||||
return self.create_error_response(
|
||||
f"No logprobs available for item {i}. "
|
||||
"This might indicate an issue with logprobs configuration."
|
||||
)
|
||||
|
||||
# The logprobs dict maps token_id -> Logprob object
|
||||
# For logprobs=-1, this contains all vocab tokens
|
||||
logprobs_dict = output.logprobs[0]
|
||||
|
||||
# Extract logprobs for label tokens
|
||||
label_logprobs: dict[int, float] = {}
|
||||
missing_tokens = []
|
||||
for token_id in request.label_token_ids:
|
||||
if token_id in logprobs_dict:
|
||||
label_logprobs[token_id] = logprobs_dict[token_id].logprob
|
||||
else:
|
||||
missing_tokens.append(token_id)
|
||||
|
||||
if missing_tokens:
|
||||
return self.create_error_response(
|
||||
f"Token IDs {missing_tokens} not found in logprobs for item {i}. "
|
||||
"This might indicate the tokens are outside the model's vocabulary."
|
||||
)
|
||||
for i, read in enumerate(reads):
|
||||
label_logprobs = dict(zip(request.label_token_ids, read.logprobs))
|
||||
|
||||
# Compute probabilities based on apply_softmax setting
|
||||
token_probs = self._compute_probabilities(
|
||||
@@ -370,7 +323,7 @@ class ServingGenerativeScoring(BaseServing):
|
||||
|
||||
# Update token counts
|
||||
total_prompt_tokens += prompt_token_counts[i]
|
||||
total_completion_tokens += len(output.token_ids)
|
||||
total_completion_tokens += len(read.result.outputs[0].token_ids)
|
||||
|
||||
# Build response
|
||||
model_name = self.models.model_name(lora_request)
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""One-token reads that return the logprobs of chosen label tokens."""
|
||||
|
||||
from collections.abc import AsyncGenerator, Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.outputs import RequestOutput
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.utils.async_utils import merge_async_iterators
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LabelRead:
|
||||
result: RequestOutput
|
||||
logprobs: list[float]
|
||||
"""Full-vocabulary logprob of each label token, in ``logprob_token_ids``
|
||||
order."""
|
||||
|
||||
|
||||
async def next_token_label_reads(
|
||||
engine_client: EngineClient,
|
||||
engine_inputs: Sequence[EngineInput],
|
||||
sampling_params: Sequence[SamplingParams],
|
||||
request_id: str,
|
||||
*,
|
||||
lora_request: LoRARequest | None = None,
|
||||
trace_headers: Mapping[str, str] | None = None,
|
||||
priority: int = 0,
|
||||
) -> list[LabelRead]:
|
||||
"""Runs one read per input at once, as request ``{request_id}-{i}``. Each
|
||||
read's params set ``max_tokens=1`` and the label tokens as
|
||||
``logprob_token_ids``. Raises ValueError naming the item when a read has
|
||||
no output or lacks a label's logprob."""
|
||||
generators: list[AsyncGenerator[RequestOutput, None]] = [
|
||||
engine_client.generate(
|
||||
engine_input,
|
||||
params,
|
||||
f"{request_id}-{i}",
|
||||
lora_request=lora_request,
|
||||
trace_headers=trace_headers,
|
||||
priority=priority,
|
||||
)
|
||||
for i, (engine_input, params) in enumerate(zip(engine_inputs, sampling_params))
|
||||
]
|
||||
results: list[RequestOutput | None] = [None] * len(generators)
|
||||
async for i, res in merge_async_iterators(*generators):
|
||||
results[i] = res
|
||||
|
||||
reads = []
|
||||
for i, (result, params) in enumerate(zip(results, sampling_params)):
|
||||
if result is None:
|
||||
raise ValueError(f"Failed to generate result for item {i}")
|
||||
if not result.outputs:
|
||||
raise ValueError(f"No output generated for item {i}")
|
||||
output = result.outputs[0]
|
||||
if output.finish_reason == "error":
|
||||
raise ValueError(f"Generation error for item {i}")
|
||||
if not output.logprobs:
|
||||
raise ValueError(
|
||||
f"No logprobs available for item {i}. "
|
||||
"This might indicate an issue with logprobs configuration."
|
||||
)
|
||||
logprobs = output.logprobs[0]
|
||||
label_ids = params.logprob_token_ids or []
|
||||
missing = [t for t in label_ids if t not in logprobs]
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"Token IDs {missing} not found in logprobs for item {i}. "
|
||||
"This might indicate the tokens are outside the model's vocabulary."
|
||||
)
|
||||
reads.append(LabelRead(result, [logprobs[t].logprob for t in label_ids]))
|
||||
return reads
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -0,0 +1,53 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
|
||||
return getattr(request.app.state, "serving_structured_decisions", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/systemone",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_structured_decision(
|
||||
request: StructuredDecisionRequest, raw_request: Request
|
||||
):
|
||||
handler = structured_decisions(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support the structured decision API"
|
||||
)
|
||||
result = await handler.create_decision(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
if getattr(app.state.args, "enable_structured_decisions", False):
|
||||
app.include_router(router)
|
||||
@@ -0,0 +1,88 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Request and response models of the structured decisions API (/v1/systemone)."""
|
||||
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.renderers import ChatParams, TokenizeParams, merge_kwargs
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
class QuestionSpec(OpenAIBaseModel):
|
||||
type: str
|
||||
instructions: Any = ""
|
||||
criteria: Any = None
|
||||
|
||||
|
||||
class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
model: str | None = None
|
||||
state: Any = Field(..., description="What the questions are about: text or JSON.")
|
||||
questions: dict[str, QuestionSpec] = Field(
|
||||
..., description="Question id to question, asked in this order."
|
||||
)
|
||||
instructions: str | None = Field(
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
request_id: str = Field(default_factory=random_uuid)
|
||||
|
||||
|
||||
class DecisionUsage(OpenAIBaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class QuestionDiagnostics(OpenAIBaseModel):
|
||||
label_mass: float = Field(
|
||||
description="Probability the model put on the labels, over the full "
|
||||
"vocabulary. A low value means most of the probability went to tokens "
|
||||
"that are not labels."
|
||||
)
|
||||
argmax_is_label: bool
|
||||
|
||||
|
||||
class StructuredDecisionResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
object: Literal["structured_decision"] = "structured_decision"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
answers: dict[str, dict[str, Any]]
|
||||
usage: DecisionUsage
|
||||
diagnostics: dict[str, QuestionDiagnostics]
|
||||
|
||||
|
||||
class ReadPromptRequest(OpenAIBaseModel):
|
||||
"""Chat options for one read's prompt: the state and the question in the
|
||||
user turn, ending at the generation prompt so the label is the reply's
|
||||
first token. Thinking is off unless the request turns it on, or the label
|
||||
would follow a thought rather than start the reply."""
|
||||
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
|
||||
def build_chat_params(
|
||||
self,
|
||||
default_template: str | None,
|
||||
default_template_content_format: ChatTemplateContentFormatOption,
|
||||
) -> ChatParams:
|
||||
return ChatParams(
|
||||
chat_template=default_template,
|
||||
chat_template_content_format=default_template_content_format,
|
||||
chat_template_kwargs=merge_kwargs(
|
||||
merge_kwargs({"enable_thinking": False}, self.chat_template_kwargs),
|
||||
dict(add_generation_prompt=True, continue_final_message=False),
|
||||
),
|
||||
)
|
||||
|
||||
def build_tok_params(self, model_config: ModelConfig) -> TokenizeParams:
|
||||
return TokenizeParams(
|
||||
max_total_tokens=model_config.max_model_len,
|
||||
max_output_tokens=1,
|
||||
add_special_tokens=False,
|
||||
)
|
||||
@@ -0,0 +1,160 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Question types for structured decisions.
|
||||
|
||||
A question type turns a request's criteria into options, writes the question
|
||||
for the model with a label per option, and builds the answer from the label
|
||||
probabilities.
|
||||
"""
|
||||
|
||||
import math
|
||||
import string
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: Options are labeled A to Z in order. A single capital letter is one token
|
||||
#: at the start of a reply for the tokenizers tested; every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Option:
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Question:
|
||||
id: str
|
||||
type: "QuestionType"
|
||||
instructions: str
|
||||
options: tuple[Option, ...]
|
||||
labels: tuple[str, ...]
|
||||
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
|
||||
@abstractmethod
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
"""The answer for ``question``. ``probs[i]`` is the probability of
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(
|
||||
f"{label}: {o.name} - {o.description}"
|
||||
if o.description
|
||||
else f"{label}: {o.name}"
|
||||
)
|
||||
lines.append("Answer with the letter of one option only.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
QUESTION_TYPES: dict[str, QuestionType] = {}
|
||||
|
||||
|
||||
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
|
||||
if cls.name in QUESTION_TYPES:
|
||||
raise ValueError(f"question type {cls.name!r} is already registered")
|
||||
QUESTION_TYPES[cls.name] = cls()
|
||||
return cls
|
||||
|
||||
|
||||
def get_question_type(name: str) -> QuestionType:
|
||||
try:
|
||||
return QUESTION_TYPES[name]
|
||||
except KeyError:
|
||||
raise StructuredDecisionError(
|
||||
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
|
||||
) from None
|
||||
|
||||
|
||||
def build_question(
|
||||
qid: str,
|
||||
type_name: str,
|
||||
instructions: Any,
|
||||
criteria: Any,
|
||||
max_options: int,
|
||||
) -> Question:
|
||||
if not qid:
|
||||
raise StructuredDecisionError("question ids must be non-empty")
|
||||
qtype = get_question_type(type_name)
|
||||
options = qtype.parse_options(qid, criteria)
|
||||
if not options:
|
||||
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(LABELS))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
)
|
||||
if not isinstance(instructions, str):
|
||||
instructions = "" if instructions is None else str(instructions)
|
||||
return Question(
|
||||
id=qid,
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def argmax(values: list[float]) -> int:
|
||||
return max(range(len(values)), key=values.__getitem__)
|
||||
|
||||
|
||||
def label_softmax(logprobs: list[float]) -> list[float]:
|
||||
"""Softmax of the label logprobs."""
|
||||
top = max(logprobs)
|
||||
weights = [math.exp(lp - top) for lp in logprobs]
|
||||
total = sum(weights)
|
||||
return [w / total for w in weights]
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ChoiceQuestion(QuestionType):
|
||||
"""Pick one option. ``criteria`` maps each option name to a description,
|
||||
or to null."""
|
||||
|
||||
name = "choice"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, dict) or not criteria:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: choice criteria must map option names to "
|
||||
"a description or null"
|
||||
)
|
||||
return [
|
||||
Option(str(name), None if desc is None else str(desc))
|
||||
for name, desc in criteria.items()
|
||||
]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
"choice": question.options[top].name,
|
||||
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
|
||||
# The chosen label's probability over the whole vocabulary, so a
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
QuestionDiagnostics,
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
from .question_types import Question, StructuredDecisionError, build_question
|
||||
from .strategies import DecisionLimits, ReadStrategy
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def state_text(state: Any) -> str:
|
||||
return state if isinstance(state, str) else json.dumps(state)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
request: StructuredDecisionRequest,
|
||||
limits: DecisionLimits,
|
||||
) -> list[Question]:
|
||||
if not request.questions:
|
||||
raise StructuredDecisionError("questions: needs at least one question")
|
||||
if len(request.questions) > limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {limits.max_questions} for this model"
|
||||
)
|
||||
questions = []
|
||||
for qid, spec in request.questions.items():
|
||||
if spec.model_extra:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
|
||||
)
|
||||
questions.append(
|
||||
build_question(
|
||||
qid,
|
||||
spec.type,
|
||||
spec.instructions,
|
||||
spec.criteria,
|
||||
limits.max_options,
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
class ServingStructuredDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
model_config = strategy.context.engine_client.model_config
|
||||
super().__init__(
|
||||
models=models, model_config=model_config, request_logger=request_logger
|
||||
)
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> StructuredDecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine_client = self.strategy.context.engine_client
|
||||
if engine_client.errored:
|
||||
raise engine_client.dead_error
|
||||
|
||||
base_id = self._base_request_id(raw_request, default=request.request_id)
|
||||
request_id = f"decision-{base_id}"
|
||||
try:
|
||||
questions = parse_questions(request, self.limits)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(len(questions))
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
state_text(request.state),
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
answers: dict[str, dict[str, Any]] = {}
|
||||
diagnostics: dict[str, QuestionDiagnostics] = {}
|
||||
for q, read in zip(questions, reads):
|
||||
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
|
||||
diagnostics[q.id] = QuestionDiagnostics(
|
||||
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
|
||||
)
|
||||
return StructuredDecisionResponse(
|
||||
id=request_id,
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=answers,
|
||||
usage=DecisionUsage(
|
||||
input_tokens=sum(r.input_tokens for r in reads),
|
||||
output_tokens=sum(r.output_tokens for r in reads),
|
||||
),
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
@@ -0,0 +1,235 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Read strategies: how a model is asked for label probabilities.
|
||||
|
||||
Only models in NEXT_TOKEN_ARCHITECTURES are currently supported.
|
||||
"""
|
||||
|
||||
import math
|
||||
from abc import ABC, abstractmethod
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.chat_utils import ChatTemplateContentFormatOption
|
||||
from vllm.entrypoints.generate.label_reads import next_token_label_reads
|
||||
from vllm.inputs import EngineInput
|
||||
from vllm.lora.request import LoRARequest
|
||||
from vllm.renderers.inputs.preprocess import extract_prompt_components
|
||||
from vllm.renderers.online_renderer import OnlineRenderer
|
||||
from vllm.sampling_params import SamplingParams
|
||||
from vllm.tokenizers import TokenizerLike
|
||||
|
||||
from .protocol import ReadPromptRequest
|
||||
from .question_types import LABELS, Question, StructuredDecisionError, label_softmax
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReadContext:
|
||||
engine_client: EngineClient
|
||||
online_renderer: OnlineRenderer
|
||||
chat_template: str | None
|
||||
chat_template_content_format: ChatTemplateContentFormatOption
|
||||
default_chat_template_kwargs: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DecisionLimits:
|
||||
max_questions: int
|
||||
max_options: int
|
||||
|
||||
|
||||
@dataclass
|
||||
class QuestionRead:
|
||||
probs: list[float]
|
||||
label_mass: float
|
||||
argmax_is_label: bool
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class ReadStrategy(ABC):
|
||||
def __init__(self, context: ReadContext):
|
||||
self.context = context
|
||||
|
||||
@abstractmethod
|
||||
def limits(self) -> DecisionLimits: ...
|
||||
|
||||
@abstractmethod
|
||||
async def read(
|
||||
self,
|
||||
questions: list[Question],
|
||||
instructions: str | None,
|
||||
state: str,
|
||||
*,
|
||||
request_id: str,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
) -> list[QuestionRead]:
|
||||
"""One read per question, in order."""
|
||||
|
||||
|
||||
def reply_label_ids(
|
||||
tokenizer: TokenizerLike, prompt_ids: Sequence[int]
|
||||
) -> tuple[list[int], list[int]]:
|
||||
"""The prompt's ids after its last added token, and the token each of LABELS
|
||||
adds after them as the first token of the reply. Raises ValueError if a
|
||||
label is not one distinct token there."""
|
||||
added = set(tokenizer.get_added_vocab().values())
|
||||
start = max((i + 1 for i, t in enumerate(prompt_ids) if t in added), default=0)
|
||||
tail = list(prompt_ids[start:])
|
||||
text = tokenizer.decode(tail)
|
||||
ids: list[int] = []
|
||||
for label in LABELS:
|
||||
extended = tokenizer.encode(text + label, add_special_tokens=False)
|
||||
if extended[:-1] != tail or extended[-1] in ids:
|
||||
raise ValueError(
|
||||
f"label {label!r} is not one distinct token after this model's "
|
||||
"chat prompt"
|
||||
)
|
||||
ids.append(extended[-1])
|
||||
return tail, ids
|
||||
|
||||
|
||||
class NextTokenStrategy(ReadStrategy):
|
||||
"""Autoregressive models. Each question is one request: the state, then the
|
||||
question with its labeled options, and the logprobs of the label tokens as
|
||||
the reply's first token. The requests share the state. With prefix caching,
|
||||
it is prefilled once."""
|
||||
|
||||
def __init__(self, context: ReadContext):
|
||||
super().__init__(context)
|
||||
tokenizer = context.online_renderer.renderer.get_tokenizer()
|
||||
probe = tokenizer.apply_chat_template(
|
||||
[{"role": "user", "content": "x"}],
|
||||
**{
|
||||
**context.default_chat_template_kwargs,
|
||||
"chat_template": context.chat_template,
|
||||
"add_generation_prompt": True,
|
||||
"tokenize": True,
|
||||
"return_dict": False,
|
||||
"enable_thinking": False,
|
||||
},
|
||||
)
|
||||
if isinstance(probe, str):
|
||||
probe = tokenizer.encode(probe, add_special_tokens=False)
|
||||
# Every prompt ends with the same generation prompt, so the labels'
|
||||
# tokens are the same for every question.
|
||||
self.tail, self.label_ids = reply_label_ids(tokenizer, probe)
|
||||
|
||||
def limits(self) -> DecisionLimits:
|
||||
return DecisionLimits(max_questions=64, max_options=len(LABELS))
|
||||
|
||||
def _read_request(
|
||||
self, chat_template_kwargs: dict[str, Any] | None
|
||||
) -> ReadPromptRequest:
|
||||
if (chat_template_kwargs or {}).get("enable_thinking"):
|
||||
raise StructuredDecisionError(
|
||||
"a read is the reply's first token, so thinking must be off"
|
||||
)
|
||||
return ReadPromptRequest(chat_template_kwargs=chat_template_kwargs)
|
||||
|
||||
async def _render(
|
||||
self, read_request: ReadPromptRequest, messages: list[dict[str, Any]]
|
||||
) -> tuple[EngineInput, list[int]]:
|
||||
ctx = self.context
|
||||
_, (engine_input,) = await ctx.online_renderer.preprocess_chat(
|
||||
read_request,
|
||||
messages,
|
||||
default_template=ctx.chat_template,
|
||||
default_template_content_format=ctx.chat_template_content_format,
|
||||
default_template_kwargs=ctx.default_chat_template_kwargs,
|
||||
)
|
||||
prompt_ids = extract_prompt_components(
|
||||
ctx.engine_client.model_config, engine_input
|
||||
).token_ids
|
||||
return engine_input, list(prompt_ids or [])
|
||||
|
||||
async def read(
|
||||
self,
|
||||
questions: list[Question],
|
||||
instructions: str | None,
|
||||
state: str,
|
||||
*,
|
||||
request_id: str,
|
||||
chat_template_kwargs: dict[str, Any] | None,
|
||||
lora_request: LoRARequest | None,
|
||||
priority: int,
|
||||
) -> list[QuestionRead]:
|
||||
ctx = self.context
|
||||
read_request = self._read_request(chat_template_kwargs)
|
||||
|
||||
slots, engine_inputs = [], []
|
||||
for q in questions:
|
||||
messages = [{"role": "user", "content": f"{state}\n\n{q.type.prompt(q)}"}]
|
||||
if instructions:
|
||||
messages.insert(0, {"role": "system", "content": instructions})
|
||||
engine_input, prompt_ids = await self._render(read_request, messages)
|
||||
if prompt_ids[len(prompt_ids) - len(self.tail) :] != self.tail:
|
||||
raise StructuredDecisionError(
|
||||
"these chat options end the prompt differently, so the "
|
||||
"labels' tokens are unknown"
|
||||
)
|
||||
slots.append(self.label_ids[: len(q.labels)])
|
||||
engine_inputs.append(engine_input)
|
||||
|
||||
label_reads = await next_token_label_reads(
|
||||
ctx.engine_client,
|
||||
engine_inputs,
|
||||
[
|
||||
SamplingParams(max_tokens=1, temperature=0.0, logprob_token_ids=ids)
|
||||
for ids in slots
|
||||
],
|
||||
request_id,
|
||||
lora_request=lora_request,
|
||||
priority=priority,
|
||||
)
|
||||
|
||||
reads = []
|
||||
for ids, label_read in zip(slots, label_reads):
|
||||
output = label_read.result.outputs[0]
|
||||
reads.append(
|
||||
QuestionRead(
|
||||
probs=label_softmax(label_read.logprobs),
|
||||
label_mass=sum(math.exp(lp) for lp in label_read.logprobs),
|
||||
argmax_is_label=bool(output.token_ids)
|
||||
and output.token_ids[0] in ids,
|
||||
input_tokens=len(label_read.result.prompt_token_ids or ()),
|
||||
output_tokens=len(output.token_ids),
|
||||
)
|
||||
)
|
||||
return reads
|
||||
|
||||
|
||||
NEXT_TOKEN_ARCHITECTURES = frozenset(
|
||||
{
|
||||
"Qwen3ForCausalLM",
|
||||
"Qwen3_5ForConditionalGeneration",
|
||||
"Qwen3_5MoeForConditionalGeneration",
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
# label_mass sums the labels' full-vocabulary probabilities. The logprobs modes
|
||||
# return those probabilities. The logits modes return raw logits instead.
|
||||
LOGPROBS_MODES = frozenset({"raw_logprobs", "processed_logprobs"})
|
||||
|
||||
|
||||
def select_read_strategy(model_config: ModelConfig) -> type[ReadStrategy]:
|
||||
"""Raises ValueError when the model cannot serve structured decisions. The
|
||||
server opted in with --enable-structured-decisions, so startup fails."""
|
||||
if model_config.architecture not in NEXT_TOKEN_ARCHITECTURES:
|
||||
raise ValueError(
|
||||
"--enable-structured-decisions does not support "
|
||||
f"{model_config.architecture}. Supported architectures: "
|
||||
f"{sorted(NEXT_TOKEN_ARCHITECTURES)}"
|
||||
)
|
||||
if model_config.logprobs_mode not in LOGPROBS_MODES:
|
||||
raise ValueError(
|
||||
"--enable-structured-decisions needs --logprobs-mode raw_logprobs or "
|
||||
f"processed_logprobs, not {model_config.logprobs_mode}"
|
||||
)
|
||||
return NextTokenStrategy
|
||||
@@ -163,6 +163,8 @@ class BaseFrontendArgs:
|
||||
enable_tokenizer_info_endpoint: bool = False
|
||||
"""Enable the `/tokenizer_info` endpoint. May expose chat
|
||||
templates and other tokenizer configuration."""
|
||||
enable_structured_decisions: bool = False
|
||||
"""Enable the `/v1/systemone` structured decisions endpoint."""
|
||||
enable_log_outputs: bool = False
|
||||
"""If set to True, log model outputs (generations). Requires
|
||||
`--enable-log-requests`. Output text and finish reasons are logged at INFO,
|
||||
|
||||
Reference in New Issue
Block a user