[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -18,8 +18,6 @@ from vllm.entrypoints.launchers.launcher import (
|
||||
setup_server,
|
||||
validate_api_server_args,
|
||||
)
|
||||
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
|
||||
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
|
||||
|
||||
warnings.warn(
|
||||
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
|
||||
@@ -35,13 +33,11 @@ __all__ = [
|
||||
"build_async_engine_client_from_engine_args",
|
||||
"build_app",
|
||||
"init_app_state",
|
||||
"init_render_app_state",
|
||||
"create_server_socket",
|
||||
"create_server_unix_socket",
|
||||
"validate_api_server_args",
|
||||
"setup_server",
|
||||
"build_and_serve",
|
||||
"build_and_serve_renderer",
|
||||
"run_server",
|
||||
"run_server_worker",
|
||||
"register_api_routers",
|
||||
|
||||
@@ -107,7 +107,6 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
|
||||
class ChatCompletionResponseChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
|
||||
# so that added fields like ``citations`` survive JSON serialization
|
||||
# instead of being stripped down to the base schema. Plain
|
||||
# ``ChatMessage`` instances serialize identically to before.
|
||||
@@ -162,7 +161,6 @@ class ChatCompletionResponse(OpenAIBaseModel):
|
||||
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
|
||||
index: int
|
||||
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
|
||||
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
|
||||
# so streaming ``citations`` survive JSON serialization. Plain
|
||||
# ``DeltaMessage`` instances serialize identically to before.
|
||||
delta: SerializeAsAny[DeltaMessage]
|
||||
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,91 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Translate Decisions questions and answers to and from label reads."""
|
||||
|
||||
import json
|
||||
|
||||
from .protocol import (
|
||||
ChoiceAnswer,
|
||||
ChoiceQuestion,
|
||||
DecisionAnswer,
|
||||
DecisionInputMessage,
|
||||
DecisionQuestion,
|
||||
PredicateAnswer,
|
||||
PredicateQuestion,
|
||||
ScoreAnswer,
|
||||
)
|
||||
from .question_types import (
|
||||
LABELS,
|
||||
Option,
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
argmax,
|
||||
)
|
||||
from .question_types import (
|
||||
ChoiceQuestion as LabelQuestion,
|
||||
)
|
||||
|
||||
|
||||
def make_read_question(
|
||||
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
|
||||
) -> Question:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
options = [Option("false"), Option("true")]
|
||||
elif isinstance(question, ChoiceQuestion):
|
||||
options = [
|
||||
Option(json.dumps(c.value, ensure_ascii=False), c.description)
|
||||
for c in question.choices
|
||||
]
|
||||
else:
|
||||
options = [Option(level.label, level.description) for level in question.levels]
|
||||
limit = min(max_options, len(LABELS))
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"This model supports at most {limit} choices per question"
|
||||
)
|
||||
return Question(
|
||||
id=str(index),
|
||||
type=LabelQuestion(),
|
||||
instructions=question.instructions,
|
||||
options=tuple(options),
|
||||
labels=LABELS[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def make_answer(
|
||||
question: DecisionQuestion, probs: list[float], label_mass: float
|
||||
) -> DecisionAnswer:
|
||||
if isinstance(question, PredicateQuestion):
|
||||
return PredicateAnswer(name=question.name, probability=probs[1])
|
||||
top = argmax(probs)
|
||||
confidence = probs[top] * label_mass
|
||||
if isinstance(question, ChoiceQuestion):
|
||||
return ChoiceAnswer(
|
||||
name=question.name,
|
||||
choice=question.choices[top].value,
|
||||
probabilities=[
|
||||
{"value": choice.value, "probability": probability}
|
||||
for choice, probability in zip(question.choices, probs)
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
return ScoreAnswer(
|
||||
name=question.name,
|
||||
score=sum(i * probability for i, probability in enumerate(probs)),
|
||||
probabilities=[
|
||||
{"value": i, "label": level.label, "probability": probability}
|
||||
for i, (level, probability) in enumerate(zip(question.levels, probs))
|
||||
],
|
||||
confidence=confidence,
|
||||
)
|
||||
|
||||
|
||||
def input_text(input: str | list[DecisionInputMessage]) -> str:
|
||||
if isinstance(input, str):
|
||||
return input
|
||||
return "\n\n".join(
|
||||
message.content
|
||||
if isinstance(message.content, str)
|
||||
else "\n".join(part.text for part in message.content)
|
||||
for message in input
|
||||
)
|
||||
@@ -1,56 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.exception_handling.error_response import (
|
||||
create_error_response,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
|
||||
from .protocol import DecisionRequest, DecisionResponse
|
||||
from .serving import OpenAIServingDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/decisions",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
response_model=DecisionResponse,
|
||||
responses={
|
||||
status.value: {"model": ErrorResponse}
|
||||
for status in (
|
||||
HTTPStatus.BAD_REQUEST,
|
||||
HTTPStatus.NOT_FOUND,
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR,
|
||||
HTTPStatus.NOT_IMPLEMENTED,
|
||||
)
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_decision(request: DecisionRequest, raw_request: Request):
|
||||
handler: OpenAIServingDecisions | None = getattr(
|
||||
raw_request.app.state, "openai_serving_decisions", None
|
||||
)
|
||||
if handler is None:
|
||||
error = create_error_response(
|
||||
NotImplementedError("The model does not support the Decisions API")
|
||||
)
|
||||
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
|
||||
result = await handler.create_decisions(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,181 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""OpenAI Decisions API request and response models."""
|
||||
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
|
||||
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
|
||||
InputText = Annotated[StrictStr, Field(max_length=10485760)]
|
||||
ChoiceValue = StrictStr | StrictBool
|
||||
|
||||
|
||||
class DecisionModel(OpenAIBaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
|
||||
class DecisionInputText(DecisionModel):
|
||||
type: Literal["input_text"]
|
||||
text: InputText
|
||||
|
||||
|
||||
class DecisionInputMessage(DecisionModel):
|
||||
role: Literal["user"]
|
||||
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
|
||||
type: Literal["message"] = "message"
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def reject_images(cls, data: Any) -> Any:
|
||||
if (
|
||||
isinstance(data, dict)
|
||||
and isinstance(data.get("content"), list)
|
||||
and any(
|
||||
isinstance(part, dict) and part.get("type") == "input_image"
|
||||
for part in data["content"]
|
||||
)
|
||||
):
|
||||
raise ValueError(
|
||||
"Decisions supports text input only; images are not supported"
|
||||
)
|
||||
return data
|
||||
|
||||
|
||||
class QuestionBase(DecisionModel):
|
||||
instructions: ShortText
|
||||
name: ShortText | None = None
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_name(self) -> "QuestionBase":
|
||||
if "name" in self.model_fields_set and self.name is None:
|
||||
raise ValueError("name must be a string when provided")
|
||||
return self
|
||||
|
||||
|
||||
class PredicateQuestion(QuestionBase):
|
||||
type: Literal["predicate"]
|
||||
|
||||
|
||||
class ChoiceOption(DecisionModel):
|
||||
value: ChoiceValue
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ChoiceQuestion(QuestionBase):
|
||||
type: Literal["choice"]
|
||||
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
|
||||
|
||||
@model_validator(mode="after")
|
||||
def validate_choices(self) -> "ChoiceQuestion":
|
||||
values = [(type(c.value), c.value) for c in self.choices]
|
||||
if len(set(values)) != len(values):
|
||||
raise ValueError("choice values must be distinct")
|
||||
return self
|
||||
|
||||
|
||||
class ScoreLevel(DecisionModel):
|
||||
label: ShortText
|
||||
description: ShortText = ""
|
||||
|
||||
|
||||
class ScoreQuestion(QuestionBase):
|
||||
type: Literal["score"]
|
||||
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
|
||||
|
||||
|
||||
DecisionQuestion = Annotated[
|
||||
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
|
||||
]
|
||||
|
||||
|
||||
class DecisionRequest(DecisionModel):
|
||||
model: ShortText
|
||||
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
|
||||
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
|
||||
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
min_length=1,
|
||||
max_length=1024,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class PredicateAnswer(DecisionModel):
|
||||
type: Literal["predicate"] = "predicate"
|
||||
name: str | None
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceProbability(DecisionModel):
|
||||
value: ChoiceValue
|
||||
probability: float
|
||||
|
||||
|
||||
class ChoiceAnswer(DecisionModel):
|
||||
type: Literal["choice"] = "choice"
|
||||
name: str | None
|
||||
choice: ChoiceValue
|
||||
probabilities: list[ChoiceProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
class ScoreProbability(DecisionModel):
|
||||
value: int
|
||||
label: str
|
||||
probability: float
|
||||
|
||||
|
||||
class ScoreAnswer(DecisionModel):
|
||||
type: Literal["score"] = "score"
|
||||
name: str | None
|
||||
score: float
|
||||
probabilities: list[ScoreProbability]
|
||||
confidence: float
|
||||
|
||||
|
||||
DecisionAnswer = Annotated[
|
||||
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
|
||||
Field(discriminator="type"),
|
||||
]
|
||||
|
||||
|
||||
class InputTokensDetails(DecisionModel):
|
||||
cached_tokens: int = 0
|
||||
cache_write_tokens: int = 0
|
||||
|
||||
|
||||
class OutputTokensDetails(DecisionModel):
|
||||
reasoning_tokens: int = 0
|
||||
|
||||
|
||||
class OpenAIDecisionUsage(DecisionModel):
|
||||
input_tokens: int
|
||||
input_tokens_details: InputTokensDetails
|
||||
output_tokens: int
|
||||
output_tokens_details: OutputTokensDetails
|
||||
total_tokens: int
|
||||
|
||||
|
||||
class DecisionResponse(DecisionModel):
|
||||
model: str
|
||||
answers: list[DecisionAnswer]
|
||||
usage: OpenAIDecisionUsage
|
||||
@@ -1,240 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Question types for structured decisions.
|
||||
|
||||
A question type turns a request's criteria into options, writes the question
|
||||
for the model with a label per option, and builds the answer from the label
|
||||
probabilities.
|
||||
"""
|
||||
|
||||
import math
|
||||
import string
|
||||
from abc import ABC, abstractmethod
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, ClassVar
|
||||
|
||||
|
||||
class StructuredDecisionError(ValueError):
|
||||
"""An invalid request. Returned as a 400."""
|
||||
|
||||
|
||||
#: A choice labels its options A to Z in order. A single capital letter is one
|
||||
#: token at the start of a reply for the tokenizers tested. Every read checks it.
|
||||
LABELS = tuple(string.ascii_uppercase)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Option:
|
||||
name: str
|
||||
description: str | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Question:
|
||||
id: str
|
||||
type: "QuestionType"
|
||||
instructions: str
|
||||
options: tuple[Option, ...]
|
||||
labels: tuple[str, ...]
|
||||
|
||||
|
||||
class QuestionType(ABC):
|
||||
name: ClassVar[str]
|
||||
#: The labels of a question's options, in option order.
|
||||
label_set: ClassVar[tuple[str, ...]] = LABELS
|
||||
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
|
||||
|
||||
@abstractmethod
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
|
||||
|
||||
@abstractmethod
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
"""The answer for ``question``. ``probs[i]`` is the probability of
|
||||
``question.labels[i]`` among the labels, and the list sums to 1.
|
||||
``label_mass`` is the labels' total probability over the vocabulary."""
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str:
|
||||
return (
|
||||
f"{label}: {option.name} - {option.description}"
|
||||
if option.description
|
||||
else f"{label}: {option.name}"
|
||||
)
|
||||
|
||||
def prompt(self, question: Question) -> str:
|
||||
"""The question as the model reads it, after the state."""
|
||||
lines = [f"Question: {question.instructions}"] if question.instructions else []
|
||||
for label, o in zip(question.labels, question.options):
|
||||
lines.append(self.reply_line(label, o))
|
||||
lines.append(self.reply_instruction)
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
QUESTION_TYPES: dict[str, QuestionType] = {}
|
||||
|
||||
|
||||
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
|
||||
if cls.name in QUESTION_TYPES:
|
||||
raise ValueError(f"question type {cls.name!r} is already registered")
|
||||
QUESTION_TYPES[cls.name] = cls()
|
||||
return cls
|
||||
|
||||
|
||||
def get_question_type(name: str) -> QuestionType:
|
||||
try:
|
||||
return QUESTION_TYPES[name]
|
||||
except KeyError:
|
||||
raise StructuredDecisionError(
|
||||
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
|
||||
) from None
|
||||
|
||||
|
||||
def build_question(
|
||||
qid: str,
|
||||
type_name: str,
|
||||
instructions: Any,
|
||||
criteria: Any,
|
||||
max_options: int,
|
||||
labeled: bool = True,
|
||||
) -> Question:
|
||||
if not qid:
|
||||
raise StructuredDecisionError("question ids must be non-empty")
|
||||
qtype = get_question_type(type_name)
|
||||
options = qtype.parse_options(qid, criteria)
|
||||
if not options:
|
||||
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
|
||||
names = [o.name for o in options]
|
||||
if len(set(names)) != len(names):
|
||||
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
|
||||
limit = min(max_options, len(qtype.label_set)) if labeled else max_options
|
||||
if len(options) > limit:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: at most {limit} options for this model"
|
||||
)
|
||||
if not isinstance(instructions, str):
|
||||
instructions = "" if instructions is None else str(instructions)
|
||||
return Question(
|
||||
id=qid,
|
||||
type=qtype,
|
||||
instructions=instructions,
|
||||
options=tuple(options),
|
||||
labels=qtype.label_set[: len(options)],
|
||||
)
|
||||
|
||||
|
||||
def argmax(values: list[float]) -> int:
|
||||
return max(range(len(values)), key=values.__getitem__)
|
||||
|
||||
|
||||
def label_softmax(logprobs: list[float]) -> list[float]:
|
||||
"""Softmax of the label logprobs."""
|
||||
top = max(logprobs)
|
||||
weights = [math.exp(lp - top) for lp in logprobs]
|
||||
total = sum(weights)
|
||||
return [w / total for w in weights]
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ChoiceQuestion(QuestionType):
|
||||
"""Pick one option. ``criteria`` maps each option name to a description,
|
||||
or to null."""
|
||||
|
||||
name = "choice"
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, dict) or not criteria:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: choice criteria must map option names to "
|
||||
"a description or null"
|
||||
)
|
||||
return [
|
||||
Option(str(name), None if desc is None else str(desc))
|
||||
for name, desc in criteria.items()
|
||||
]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
"choice": question.options[top].name,
|
||||
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
|
||||
# The chosen label's probability over the whole vocabulary, so a
|
||||
# read that mostly wanted a non-label token reports low confidence.
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class NoulQuestion(QuestionType):
|
||||
"""Yes or no. ``criteria`` may describe what true and false mean."""
|
||||
|
||||
name = "noul"
|
||||
label_set = ("yes", "no")
|
||||
reply_instruction = "Answer with yes or no only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if criteria is not None and (
|
||||
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
|
||||
):
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: noul criteria must be an object with true and false"
|
||||
)
|
||||
criteria = criteria or {}
|
||||
true = criteria.get("true")
|
||||
false = criteria.get("false")
|
||||
return [
|
||||
Option("yes", None if true is None else str(true)),
|
||||
Option("no", None if false is None else str(false)),
|
||||
]
|
||||
|
||||
def reply_line(self, label: str, option: Option) -> str:
|
||||
# A side without a description still shows its label. Without the
|
||||
# label in the prompt, models answer "Yes" or "No" instead.
|
||||
return f"{label}: {option.description}" if option.description else label
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"type": self.name,
|
||||
"noul": probs[0],
|
||||
"probabilities": {"yes": probs[0], "no": probs[1]},
|
||||
# The chosen label's probability over the whole vocabulary, like a
|
||||
# choice's confidence.
|
||||
"confidence": max(probs) * label_mass,
|
||||
}
|
||||
|
||||
|
||||
@register_question_type
|
||||
class ScoreQuestion(QuestionType):
|
||||
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
|
||||
level names, from the first to the last level of the scale."""
|
||||
|
||||
name = "score"
|
||||
# 0-indexed like the answer, so a level's label is its score.
|
||||
label_set = tuple(string.digits)
|
||||
reply_instruction = "Answer with the number of one level only."
|
||||
|
||||
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
|
||||
if not isinstance(criteria, list) or len(criteria) < 2:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: score criteria must be an ordered list of levels"
|
||||
)
|
||||
return [Option(str(level)) for level in criteria]
|
||||
|
||||
def answer(
|
||||
self, question: Question, probs: list[float], label_mass: float
|
||||
) -> dict[str, Any]:
|
||||
top = argmax(probs)
|
||||
return {
|
||||
"type": self.name,
|
||||
# The expected level, 0-indexed, so a level's index in the request's
|
||||
# criteria is its value.
|
||||
"score": sum(i * p for i, p in enumerate(probs)),
|
||||
"legend": {str(i): o.name for i, o in enumerate(question.options)},
|
||||
"probabilities": {str(i): p for i, p in enumerate(probs)},
|
||||
"confidence": probs[top] * label_mass,
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
|
||||
from .adapters import input_text, make_answer, make_read_question
|
||||
from .protocol import (
|
||||
DecisionRequest,
|
||||
DecisionResponse,
|
||||
InputTokensDetails,
|
||||
OpenAIDecisionUsage,
|
||||
OutputTokensDetails,
|
||||
)
|
||||
from .question_types import StructuredDecisionError
|
||||
from .strategies import ReadStrategy
|
||||
|
||||
|
||||
class BaseServingDecisions(BaseServing):
|
||||
def __init__(
|
||||
self,
|
||||
models: OpenAIServingModels,
|
||||
strategy: ReadStrategy,
|
||||
*,
|
||||
request_logger: RequestLogger | None = None,
|
||||
) -> None:
|
||||
super().__init__(
|
||||
models=models,
|
||||
model_config=strategy.context.engine_client.model_config,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
self.strategy = strategy
|
||||
self.limits = strategy.limits()
|
||||
|
||||
|
||||
class OpenAIServingDecisions(BaseServingDecisions):
|
||||
async def create_decisions(
|
||||
self, request: DecisionRequest, raw_request: Request | None = None
|
||||
) -> DecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine = self.strategy.context.engine_client
|
||||
if engine.errored:
|
||||
raise engine.dead_error
|
||||
request_id = f"decision-{self._base_request_id(raw_request)}"
|
||||
try:
|
||||
if len(request.questions) > self.limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {self.limits.max_questions} for this model"
|
||||
)
|
||||
questions = [
|
||||
make_read_question(i, question, self.limits.max_options)
|
||||
for i, question in enumerate(request.questions)
|
||||
]
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine.check_admission(len(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
text = input_text(request.input)
|
||||
self._log_inputs(request_id, text, None, lora_request)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
None,
|
||||
text,
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=None,
|
||||
lora_request=lora_request,
|
||||
priority=0,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
)
|
||||
except StructuredDecisionError as exc:
|
||||
return self.create_error_response(exc)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
input_tokens = sum(read.input_tokens for read in reads)
|
||||
output_tokens = sum(read.output_tokens for read in reads)
|
||||
return DecisionResponse(
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=[
|
||||
make_answer(question, read.probs, read.label_mass)
|
||||
for question, read in zip(request.questions, reads)
|
||||
],
|
||||
usage=OpenAIDecisionUsage(
|
||||
input_tokens=input_tokens,
|
||||
output_tokens=output_tokens,
|
||||
total_tokens=input_tokens + output_tokens,
|
||||
input_tokens_details=InputTokensDetails(
|
||||
cached_tokens=sum(read.cached_tokens for read in reads),
|
||||
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
|
||||
),
|
||||
output_tokens_details=OutputTokensDetails(),
|
||||
),
|
||||
)
|
||||
@@ -1,46 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from typing import Any
|
||||
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .serving import OpenAIServingDecisions
|
||||
from .strategies import ReadContext, ReadStrategy, select_read_strategy
|
||||
|
||||
|
||||
def init_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
chat_template: str | None,
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
) -> ReadStrategy | None:
|
||||
state.openai_serving_decisions = None
|
||||
try:
|
||||
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
|
||||
strategy = strategy_cls(
|
||||
ReadContext(
|
||||
engine_client=engine_client,
|
||||
online_renderer=state.online_renderer,
|
||||
chat_template=chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
)
|
||||
)
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
if strategy.task == "generate":
|
||||
state.openai_serving_decisions = OpenAIServingDecisions(
|
||||
state.openai_serving_models,
|
||||
strategy,
|
||||
request_logger=request_logger,
|
||||
)
|
||||
return strategy
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,37 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import warnings
|
||||
|
||||
from prometheus_client import start_http_server
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.version import __version__ as VLLM_VERSION
|
||||
|
||||
warnings.warn(
|
||||
"The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
|
||||
"and may be removed in a future release. Please use `vllm run_batch` instead.",
|
||||
DeprecationWarning,
|
||||
stacklevel=1,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
from vllm.entrypoints.launchers.run_batch import main, parse_args
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
args = parse_args()
|
||||
|
||||
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
|
||||
logger.info("args: %s", args)
|
||||
|
||||
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
|
||||
# to publish metrics at the /metrics endpoint.
|
||||
if args.enable_metrics:
|
||||
logger.info("Prometheus metrics enabled")
|
||||
start_http_server(port=args.port, addr=args.host)
|
||||
else:
|
||||
logger.info("Prometheus metrics disabled")
|
||||
|
||||
asyncio.run(main(args))
|
||||
Reference in New Issue
Block a user