[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack

- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions,
  rl, render-launcher and batch entrypoints; keep scale-out token routes
- Drop the Swagger static bundle and the /docs, /redoc routes
- Delete device-keyed tuning configs for fused_moe, quant utils, mamba
  select-state and helion; no sm86 entry ever existed for any of them
- Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant
  quantizers and their config, metrics and loader plumbing
- Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu
  requirements and the setup.py cpu target path
- Delete 29 environment variables nothing reads and 14 chat-template
  fallbacks for models this fork no longer registers
- Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark,
  dsv3 gemm) with their kernels, bindings, wrappers and CMake entries
- Delete the connector test suites that targeted removed connectors
- Cut the Mistral tokenizer and parser stack, its tests, and the
  mistral_common and ijson dependency pins
- Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier
  cut deleted while keeping the model tree

Co-authored-by: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-19 17:10:56 +07:00
co-authored by Hinata
parent bd70a61c20
commit 2066d3143f
1086 changed files with 122 additions and 291919 deletions
-4
View File
@@ -18,8 +18,6 @@ from vllm.entrypoints.launchers.launcher import (
setup_server,
validate_api_server_args,
)
from vllm.entrypoints.launchers.render.app_state import init_render_app_state
from vllm.entrypoints.launchers.render.entry import build_and_serve_renderer
warnings.warn(
"`vllm.entrypoints.openai.api_server` is deprecated and will likely be"
@@ -35,13 +33,11 @@ __all__ = [
"build_async_engine_client_from_engine_args",
"build_app",
"init_app_state",
"init_render_app_state",
"create_server_socket",
"create_server_unix_socket",
"validate_api_server_args",
"setup_server",
"build_and_serve",
"build_and_serve_renderer",
"run_server",
"run_server_worker",
"register_api_routers",
@@ -107,7 +107,6 @@ class ChatCompletionLogProbs(OpenAIBaseModel):
class ChatCompletionResponseChoice(OpenAIBaseModel):
index: int
# ``SerializeAsAny`` lets pydantic honor subclasses of ``ChatMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereChatMessage``)
# so that added fields like ``citations`` survive JSON serialization
# instead of being stripped down to the base schema. Plain
# ``ChatMessage`` instances serialize identically to before.
@@ -162,7 +161,6 @@ class ChatCompletionResponse(OpenAIBaseModel):
class ChatCompletionResponseStreamChoice(OpenAIBaseModel):
index: int
# ``SerializeAsAny`` lets pydantic honor subclasses of ``DeltaMessage``
# (e.g. ``vllm.entrypoints.cohere.cohere_chat_message.CohereDeltaMessage``)
# so streaming ``citations`` survive JSON serialization. Plain
# ``DeltaMessage`` instances serialize identically to before.
delta: SerializeAsAny[DeltaMessage]
@@ -1,2 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
@@ -1,91 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Translate Decisions questions and answers to and from label reads."""
import json
from .protocol import (
ChoiceAnswer,
ChoiceQuestion,
DecisionAnswer,
DecisionInputMessage,
DecisionQuestion,
PredicateAnswer,
PredicateQuestion,
ScoreAnswer,
)
from .question_types import (
LABELS,
Option,
Question,
StructuredDecisionError,
argmax,
)
from .question_types import (
ChoiceQuestion as LabelQuestion,
)
def make_read_question(
index: int, question: DecisionQuestion, max_options: int = len(LABELS)
) -> Question:
if isinstance(question, PredicateQuestion):
options = [Option("false"), Option("true")]
elif isinstance(question, ChoiceQuestion):
options = [
Option(json.dumps(c.value, ensure_ascii=False), c.description)
for c in question.choices
]
else:
options = [Option(level.label, level.description) for level in question.levels]
limit = min(max_options, len(LABELS))
if len(options) > limit:
raise StructuredDecisionError(
f"This model supports at most {limit} choices per question"
)
return Question(
id=str(index),
type=LabelQuestion(),
instructions=question.instructions,
options=tuple(options),
labels=LABELS[: len(options)],
)
def make_answer(
question: DecisionQuestion, probs: list[float], label_mass: float
) -> DecisionAnswer:
if isinstance(question, PredicateQuestion):
return PredicateAnswer(name=question.name, probability=probs[1])
top = argmax(probs)
confidence = probs[top] * label_mass
if isinstance(question, ChoiceQuestion):
return ChoiceAnswer(
name=question.name,
choice=question.choices[top].value,
probabilities=[
{"value": choice.value, "probability": probability}
for choice, probability in zip(question.choices, probs)
],
confidence=confidence,
)
return ScoreAnswer(
name=question.name,
score=sum(i * probability for i, probability in enumerate(probs)),
probabilities=[
{"value": i, "label": level.label, "probability": probability}
for i, (level, probability) in enumerate(zip(question.levels, probs))
],
confidence=confidence,
)
def input_text(input: str | list[DecisionInputMessage]) -> str:
if isinstance(input, str):
return input
return "\n\n".join(
message.content
if isinstance(message.content, str)
else "\n".join(part.text for part in message.content)
for message in input
)
@@ -1,56 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from http import HTTPStatus
from fastapi import APIRouter, Depends, FastAPI, Request
from fastapi.responses import JSONResponse
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.exception_handling.error_response import (
create_error_response,
)
from vllm.entrypoints.serve.utils.api_utils import (
load_aware_call,
validate_json_request,
with_cancellation,
)
from .protocol import DecisionRequest, DecisionResponse
from .serving import OpenAIServingDecisions
router = APIRouter()
@router.post(
"/v1/decisions",
dependencies=[Depends(validate_json_request)],
response_model=DecisionResponse,
responses={
status.value: {"model": ErrorResponse}
for status in (
HTTPStatus.BAD_REQUEST,
HTTPStatus.NOT_FOUND,
HTTPStatus.INTERNAL_SERVER_ERROR,
HTTPStatus.NOT_IMPLEMENTED,
)
},
)
@with_cancellation
@load_aware_call
async def create_decision(request: DecisionRequest, raw_request: Request):
handler: OpenAIServingDecisions | None = getattr(
raw_request.app.state, "openai_serving_decisions", None
)
if handler is None:
error = create_error_response(
NotImplementedError("The model does not support the Decisions API")
)
return JSONResponse(content=error.model_dump(), status_code=error.error.code)
result = await handler.create_decisions(request, raw_request)
if isinstance(result, ErrorResponse):
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
return JSONResponse(content=result.model_dump())
def register_decisions_api_router(app: FastAPI):
app.include_router(router)
@@ -1,181 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""OpenAI Decisions API request and response models."""
from typing import Annotated, Any, Literal
from pydantic import ConfigDict, Field, StrictBool, StrictStr, model_validator
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
ShortText = Annotated[StrictStr, Field(max_length=1048576)]
InputText = Annotated[StrictStr, Field(max_length=10485760)]
ChoiceValue = StrictStr | StrictBool
class DecisionModel(OpenAIBaseModel):
model_config = ConfigDict(extra="forbid")
class DecisionInputText(DecisionModel):
type: Literal["input_text"]
text: InputText
class DecisionInputMessage(DecisionModel):
role: Literal["user"]
content: InputText | Annotated[list[DecisionInputText], Field(max_length=16384)]
type: Literal["message"] = "message"
@model_validator(mode="before")
@classmethod
def reject_images(cls, data: Any) -> Any:
if (
isinstance(data, dict)
and isinstance(data.get("content"), list)
and any(
isinstance(part, dict) and part.get("type") == "input_image"
for part in data["content"]
)
):
raise ValueError(
"Decisions supports text input only; images are not supported"
)
return data
class QuestionBase(DecisionModel):
instructions: ShortText
name: ShortText | None = None
@model_validator(mode="after")
def validate_name(self) -> "QuestionBase":
if "name" in self.model_fields_set and self.name is None:
raise ValueError("name must be a string when provided")
return self
class PredicateQuestion(QuestionBase):
type: Literal["predicate"]
class ChoiceOption(DecisionModel):
value: ChoiceValue
description: ShortText = ""
class ChoiceQuestion(QuestionBase):
type: Literal["choice"]
choices: list[ChoiceOption] = Field(min_length=2, max_length=255)
@model_validator(mode="after")
def validate_choices(self) -> "ChoiceQuestion":
values = [(type(c.value), c.value) for c in self.choices]
if len(set(values)) != len(values):
raise ValueError("choice values must be distinct")
return self
class ScoreLevel(DecisionModel):
label: ShortText
description: ShortText = ""
class ScoreQuestion(QuestionBase):
type: Literal["score"]
levels: list[ScoreLevel] = Field(min_length=2, max_length=10)
DecisionQuestion = Annotated[
PredicateQuestion | ChoiceQuestion | ScoreQuestion, Field(discriminator="type")
]
class DecisionRequest(DecisionModel):
model: ShortText
input: InputText | Annotated[list[DecisionInputMessage], Field(max_length=131072)]
questions: list[DecisionQuestion] = Field(min_length=1, max_length=200)
safety_identifier: Annotated[StrictStr, Field(max_length=128)] | None = None
cache_salt: str | None = Field(
default=None,
min_length=1,
max_length=1024,
description=(
"If specified, the prefix cache will be salted with the provided "
"string to prevent an attacker to guess prompts in multi-user "
"environments. The salt should be random, protected from "
"access by 3rd parties, and long enough to be "
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
"to 256 bit)."
),
)
@model_validator(mode="before")
@classmethod
def check_cache_salt_support(cls, data: Any) -> Any:
if isinstance(data, dict):
validate_cache_salt(data.get("cache_salt"))
return data
class PredicateAnswer(DecisionModel):
type: Literal["predicate"] = "predicate"
name: str | None
probability: float
class ChoiceProbability(DecisionModel):
value: ChoiceValue
probability: float
class ChoiceAnswer(DecisionModel):
type: Literal["choice"] = "choice"
name: str | None
choice: ChoiceValue
probabilities: list[ChoiceProbability]
confidence: float
class ScoreProbability(DecisionModel):
value: int
label: str
probability: float
class ScoreAnswer(DecisionModel):
type: Literal["score"] = "score"
name: str | None
score: float
probabilities: list[ScoreProbability]
confidence: float
DecisionAnswer = Annotated[
PredicateAnswer | ChoiceAnswer | ScoreAnswer,
Field(discriminator="type"),
]
class InputTokensDetails(DecisionModel):
cached_tokens: int = 0
cache_write_tokens: int = 0
class OutputTokensDetails(DecisionModel):
reasoning_tokens: int = 0
class OpenAIDecisionUsage(DecisionModel):
input_tokens: int
input_tokens_details: InputTokensDetails
output_tokens: int
output_tokens_details: OutputTokensDetails
total_tokens: int
class DecisionResponse(DecisionModel):
model: str
answers: list[DecisionAnswer]
usage: OpenAIDecisionUsage
@@ -1,240 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Question types for structured decisions.
A question type turns a request's criteria into options, writes the question
for the model with a label per option, and builds the answer from the label
probabilities.
"""
import math
import string
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any, ClassVar
class StructuredDecisionError(ValueError):
"""An invalid request. Returned as a 400."""
#: A choice labels its options A to Z in order. A single capital letter is one
#: token at the start of a reply for the tokenizers tested. Every read checks it.
LABELS = tuple(string.ascii_uppercase)
@dataclass(frozen=True)
class Option:
name: str
description: str | None = None
@dataclass(frozen=True)
class Question:
id: str
type: "QuestionType"
instructions: str
options: tuple[Option, ...]
labels: tuple[str, ...]
class QuestionType(ABC):
name: ClassVar[str]
#: The labels of a question's options, in option order.
label_set: ClassVar[tuple[str, ...]] = LABELS
reply_instruction: ClassVar[str] = "Answer with the letter of one option only."
@abstractmethod
def parse_options(self, qid: str, criteria: Any) -> list[Option]: ...
@abstractmethod
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
"""The answer for ``question``. ``probs[i]`` is the probability of
``question.labels[i]`` among the labels, and the list sums to 1.
``label_mass`` is the labels' total probability over the vocabulary."""
def reply_line(self, label: str, option: Option) -> str:
return (
f"{label}: {option.name} - {option.description}"
if option.description
else f"{label}: {option.name}"
)
def prompt(self, question: Question) -> str:
"""The question as the model reads it, after the state."""
lines = [f"Question: {question.instructions}"] if question.instructions else []
for label, o in zip(question.labels, question.options):
lines.append(self.reply_line(label, o))
lines.append(self.reply_instruction)
return "\n".join(lines)
QUESTION_TYPES: dict[str, QuestionType] = {}
def register_question_type(cls: type[QuestionType]) -> type[QuestionType]:
if cls.name in QUESTION_TYPES:
raise ValueError(f"question type {cls.name!r} is already registered")
QUESTION_TYPES[cls.name] = cls()
return cls
def get_question_type(name: str) -> QuestionType:
try:
return QUESTION_TYPES[name]
except KeyError:
raise StructuredDecisionError(
f"unknown question type {name!r}; supported: {sorted(QUESTION_TYPES)}"
) from None
def build_question(
qid: str,
type_name: str,
instructions: Any,
criteria: Any,
max_options: int,
labeled: bool = True,
) -> Question:
if not qid:
raise StructuredDecisionError("question ids must be non-empty")
qtype = get_question_type(type_name)
options = qtype.parse_options(qid, criteria)
if not options:
raise StructuredDecisionError(f"question {qid!r}: needs at least one option")
names = [o.name for o in options]
if len(set(names)) != len(names):
raise StructuredDecisionError(f"question {qid!r}: duplicate option names")
limit = min(max_options, len(qtype.label_set)) if labeled else max_options
if len(options) > limit:
raise StructuredDecisionError(
f"question {qid!r}: at most {limit} options for this model"
)
if not isinstance(instructions, str):
instructions = "" if instructions is None else str(instructions)
return Question(
id=qid,
type=qtype,
instructions=instructions,
options=tuple(options),
labels=qtype.label_set[: len(options)],
)
def argmax(values: list[float]) -> int:
return max(range(len(values)), key=values.__getitem__)
def label_softmax(logprobs: list[float]) -> list[float]:
"""Softmax of the label logprobs."""
top = max(logprobs)
weights = [math.exp(lp - top) for lp in logprobs]
total = sum(weights)
return [w / total for w in weights]
@register_question_type
class ChoiceQuestion(QuestionType):
"""Pick one option. ``criteria`` maps each option name to a description,
or to null."""
name = "choice"
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, dict) or not criteria:
raise StructuredDecisionError(
f"question {qid!r}: choice criteria must map option names to "
"a description or null"
)
return [
Option(str(name), None if desc is None else str(desc))
for name, desc in criteria.items()
]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
"choice": question.options[top].name,
"probabilities": {a.name: p for a, p in zip(question.options, probs)},
# The chosen label's probability over the whole vocabulary, so a
# read that mostly wanted a non-label token reports low confidence.
"confidence": probs[top] * label_mass,
}
@register_question_type
class NoulQuestion(QuestionType):
"""Yes or no. ``criteria`` may describe what true and false mean."""
name = "noul"
label_set = ("yes", "no")
reply_instruction = "Answer with yes or no only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if criteria is not None and (
not isinstance(criteria, dict) or not criteria.keys() <= {"true", "false"}
):
raise StructuredDecisionError(
f"question {qid!r}: noul criteria must be an object with true and false"
)
criteria = criteria or {}
true = criteria.get("true")
false = criteria.get("false")
return [
Option("yes", None if true is None else str(true)),
Option("no", None if false is None else str(false)),
]
def reply_line(self, label: str, option: Option) -> str:
# A side without a description still shows its label. Without the
# label in the prompt, models answer "Yes" or "No" instead.
return f"{label}: {option.description}" if option.description else label
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
return {
"type": self.name,
"noul": probs[0],
"probabilities": {"yes": probs[0], "no": probs[1]},
# The chosen label's probability over the whole vocabulary, like a
# choice's confidence.
"confidence": max(probs) * label_mass,
}
@register_question_type
class ScoreQuestion(QuestionType):
"""Rate the state on an ordered scale. ``criteria`` is the ordered list of
level names, from the first to the last level of the scale."""
name = "score"
# 0-indexed like the answer, so a level's label is its score.
label_set = tuple(string.digits)
reply_instruction = "Answer with the number of one level only."
def parse_options(self, qid: str, criteria: Any) -> list[Option]:
if not isinstance(criteria, list) or len(criteria) < 2:
raise StructuredDecisionError(
f"question {qid!r}: score criteria must be an ordered list of levels"
)
return [Option(str(level)) for level in criteria]
def answer(
self, question: Question, probs: list[float], label_mass: float
) -> dict[str, Any]:
top = argmax(probs)
return {
"type": self.name,
# The expected level, 0-indexed, so a level's index in the request's
# criteria is its value.
"score": sum(i * p for i, p in enumerate(probs)),
"legend": {str(i): o.name for i, o in enumerate(question.options)},
"probabilities": {str(i): p for i, p in enumerate(probs)},
"confidence": probs[top] * label_mass,
}
@@ -1,102 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
from fastapi import Request
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
from vllm.entrypoints.serve.engine.serving import BaseServing
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from .adapters import input_text, make_answer, make_read_question
from .protocol import (
DecisionRequest,
DecisionResponse,
InputTokensDetails,
OpenAIDecisionUsage,
OutputTokensDetails,
)
from .question_types import StructuredDecisionError
from .strategies import ReadStrategy
class BaseServingDecisions(BaseServing):
def __init__(
self,
models: OpenAIServingModels,
strategy: ReadStrategy,
*,
request_logger: RequestLogger | None = None,
) -> None:
super().__init__(
models=models,
model_config=strategy.context.engine_client.model_config,
request_logger=request_logger,
)
self.strategy = strategy
self.limits = strategy.limits()
class OpenAIServingDecisions(BaseServingDecisions):
async def create_decisions(
self, request: DecisionRequest, raw_request: Request | None = None
) -> DecisionResponse | ErrorResponse:
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
return error
engine = self.strategy.context.engine_client
if engine.errored:
raise engine.dead_error
request_id = f"decision-{self._base_request_id(raw_request)}"
try:
if len(request.questions) > self.limits.max_questions:
raise StructuredDecisionError(
f"questions: at most {self.limits.max_questions} for this model"
)
questions = [
make_read_question(i, question, self.limits.max_options)
for i, question in enumerate(request.questions)
]
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
engine.check_admission(len(questions))
trace_headers = (
None
if raw_request is None
else await self._get_trace_headers(raw_request.headers)
)
text = input_text(request.input)
self._log_inputs(request_id, text, None, lora_request)
reads = await self.strategy.read(
questions,
None,
text,
request_id=request_id,
chat_template_kwargs=None,
lora_request=lora_request,
priority=0,
cache_salt=request.cache_salt,
trace_headers=trace_headers,
)
except StructuredDecisionError as exc:
return self.create_error_response(exc)
except asyncio.CancelledError:
return self.create_error_response("Client disconnected")
input_tokens = sum(read.input_tokens for read in reads)
output_tokens = sum(read.output_tokens for read in reads)
return DecisionResponse(
model=self.models.model_name(lora_request),
answers=[
make_answer(question, read.probs, read.label_mass)
for question, read in zip(request.questions, reads)
],
usage=OpenAIDecisionUsage(
input_tokens=input_tokens,
output_tokens=output_tokens,
total_tokens=input_tokens + output_tokens,
input_tokens_details=InputTokensDetails(
cached_tokens=sum(read.cached_tokens for read in reads),
cache_write_tokens=sum(read.cache_write_tokens for read in reads),
),
output_tokens_details=OutputTokensDetails(),
),
)
@@ -1,46 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from argparse import Namespace
from typing import Any
from starlette.datastructures import State
from vllm.engine.protocol import EngineClient
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
from vllm.tasks import SupportedTask
from .serving import OpenAIServingDecisions
from .strategies import ReadContext, ReadStrategy, select_read_strategy
def init_decisions_state(
engine_client: EngineClient,
state: State,
args: Namespace,
request_logger: RequestLogger | None,
chat_template: str | None,
default_chat_template_kwargs: dict[str, Any],
supported_tasks: tuple[SupportedTask, ...],
) -> ReadStrategy | None:
state.openai_serving_decisions = None
try:
strategy_cls = select_read_strategy(engine_client.model_config, supported_tasks)
strategy = strategy_cls(
ReadContext(
engine_client=engine_client,
online_renderer=state.online_renderer,
chat_template=chat_template,
chat_template_content_format=args.chat_template_content_format,
default_chat_template_kwargs=default_chat_template_kwargs,
)
)
except ValueError:
return None
if strategy.task == "generate":
state.openai_serving_decisions = OpenAIServingDecisions(
state.openai_serving_models,
strategy,
request_logger=request_logger,
)
return strategy
File diff suppressed because it is too large Load Diff
-37
View File
@@ -1,37 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import asyncio
import warnings
from prometheus_client import start_http_server
from vllm.logger import init_logger
from vllm.version import __version__ as VLLM_VERSION
warnings.warn(
"The `python -m vllm.entrypoints.openai.run_batch` command is deprecated "
"and may be removed in a future release. Please use `vllm run_batch` instead.",
DeprecationWarning,
stacklevel=1,
)
if __name__ == "__main__":
from vllm.entrypoints.launchers.run_batch import main, parse_args
logger = init_logger(__name__)
args = parse_args()
logger.info("vLLM batch processing API version %s", VLLM_VERSION)
logger.info("args: %s", args)
# Start the Prometheus metrics server. LLMEngine uses the Prometheus client
# to publish metrics at the /metrics endpoint.
if args.enable_metrics:
logger.info("Prometheus metrics enabled")
start_http_server(port=args.port, addr=args.host)
else:
logger.info("Prometheus metrics disabled")
asyncio.run(main(args))