[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -4,8 +4,6 @@ from typing import TYPE_CHECKING, Any
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
import vllm.envs as envs
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from argparse import Namespace
|
||||
|
||||
@@ -37,17 +35,6 @@ def register_generate_api_routers(app: FastAPI):
|
||||
|
||||
register_completion_api_router(app)
|
||||
|
||||
from vllm.entrypoints.anthropic.api_router import (
|
||||
attach_router as register_anthropic_api_router,
|
||||
)
|
||||
|
||||
register_anthropic_api_router(app)
|
||||
|
||||
from vllm.entrypoints.cohere.api_router import (
|
||||
attach_router as register_cohere_api_router,
|
||||
)
|
||||
|
||||
register_cohere_api_router(app)
|
||||
|
||||
from .generative_scoring.api_router import register_generative_scoring_api_router
|
||||
|
||||
@@ -62,24 +49,6 @@ async def init_generate_state(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
from vllm.entrypoints.anthropic.serving import AnthropicServingMessages
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
|
||||
# The Cohere serving handler depends on the optional `cohere` SDK for
|
||||
# its wire-format protocol models, and is additionally gated on the
|
||||
# `VLLM_ENABLE_COHERE_API` env flag (see
|
||||
# `vllm.entrypoints.cohere.api_router.attach_router`). Skip the import
|
||||
# entirely when the endpoint isn't going to be exposed, both because
|
||||
# the SDK may not be installed and because the serving object holds
|
||||
# nontrivial state (chat handler, warmup) that would otherwise be
|
||||
# unused.
|
||||
if envs.VLLM_ENABLE_COHERE_API:
|
||||
try:
|
||||
from vllm.entrypoints.cohere.serving import CohereServingChatV2
|
||||
except ImportError:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
else:
|
||||
CohereServingChatV2 = None # type: ignore[assignment,misc]
|
||||
from vllm.entrypoints.openai.chat_completion.batch_serving import (
|
||||
OpenAIServingChatBatch,
|
||||
)
|
||||
@@ -87,6 +56,7 @@ async def init_generate_state(
|
||||
from vllm.entrypoints.openai.completion.serving import OpenAIServingCompletion
|
||||
from vllm.entrypoints.openai.responses.serving import OpenAIServingResponses
|
||||
from vllm.entrypoints.serve.utils.fingerprint import set_default_fingerprint_mode
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
|
||||
# Applied before any serving class is constructed so that each one picks
|
||||
# up the chosen mode on its first cache miss.
|
||||
@@ -168,54 +138,6 @@ async def init_generate_state(
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.anthropic_serving_messages = (
|
||||
AnthropicServingMessages(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
disabled_thinking_effort=args.anthropic_disabled_thinking_effort,
|
||||
)
|
||||
if "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
state.cohere_serving_chat_v2 = (
|
||||
CohereServingChatV2(
|
||||
engine_client,
|
||||
state.openai_serving_models,
|
||||
args.response_role,
|
||||
online_renderer=state.online_renderer,
|
||||
request_logger=request_logger,
|
||||
chat_template=resolved_chat_template,
|
||||
chat_template_content_format=args.chat_template_content_format,
|
||||
return_tokens_as_token_ids=args.return_tokens_as_token_ids,
|
||||
enable_auto_tools=args.enable_auto_tool_choice,
|
||||
tool_parser=args.tool_call_parser,
|
||||
tool_strict_level=args.tool_strict_level,
|
||||
reasoning_parser=args.structured_outputs_config.reasoning_parser,
|
||||
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
|
||||
enable_force_include_usage=args.enable_force_include_usage,
|
||||
enable_log_outputs=args.enable_log_outputs,
|
||||
enable_log_deltas=args.enable_log_deltas,
|
||||
default_chat_template_kwargs=default_chat_template_kwargs,
|
||||
is_reasoning_model=args.cohere_is_reasoning_model,
|
||||
)
|
||||
if CohereServingChatV2 is not None and "generate" in supported_tasks
|
||||
else None
|
||||
)
|
||||
|
||||
from .generative_scoring.serving import ServingGenerativeScoring
|
||||
|
||||
|
||||
@@ -1,42 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from vllm.entrypoints.serve.sagemaker.api_router import (
|
||||
EndpointFn,
|
||||
GetHandlerFn,
|
||||
RequestType,
|
||||
)
|
||||
|
||||
|
||||
def get_generate_invocation_types(
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
# NOTE: Items defined earlier take higher priority
|
||||
invocation_types: list[tuple[RequestType, tuple[GetHandlerFn, EndpointFn]]] = []
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.openai.chat_completion.api_router import (
|
||||
chat,
|
||||
create_chat_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.chat_completion.protocol import (
|
||||
ChatCompletionRequest,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.api_router import (
|
||||
completion,
|
||||
create_completion,
|
||||
)
|
||||
from vllm.entrypoints.openai.completion.protocol import CompletionRequest
|
||||
|
||||
invocation_types += [
|
||||
(ChatCompletionRequest, (chat, create_chat_completion)),
|
||||
(CompletionRequest, (completion, create_completion)),
|
||||
]
|
||||
|
||||
return invocation_types
|
||||
@@ -1,2 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
@@ -1,86 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from argparse import Namespace
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
from starlette.datastructures import State
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.openai.decisions.state import init_decisions_state
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import (
|
||||
load_aware_call,
|
||||
validate_json_request,
|
||||
with_cancellation,
|
||||
)
|
||||
from vllm.entrypoints.serve.utils.request_logger import RequestLogger
|
||||
from vllm.renderers.chat_utils import load_chat_template
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
from .protocol import StructuredDecisionRequest
|
||||
from .serving import ServingStructuredDecisions
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def structured_decisions(request: Request) -> ServingStructuredDecisions | None:
|
||||
return getattr(request.app.state, "serving_structured_decisions", None)
|
||||
|
||||
|
||||
@router.post(
|
||||
"/v1/systemone",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_FOUND.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
HTTPStatus.NOT_IMPLEMENTED.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@with_cancellation
|
||||
@load_aware_call
|
||||
async def create_structured_decision(
|
||||
request: StructuredDecisionRequest, raw_request: Request
|
||||
):
|
||||
handler = structured_decisions(raw_request)
|
||||
if handler is None:
|
||||
raise NotImplementedError(
|
||||
"The model does not support the structured decision API"
|
||||
)
|
||||
result = await handler.create_decision(request, raw_request)
|
||||
if isinstance(result, ErrorResponse):
|
||||
return JSONResponse(content=result.model_dump(), status_code=result.error.code)
|
||||
return JSONResponse(content=result.model_dump())
|
||||
|
||||
|
||||
def register_structured_decisions_api_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def init_structured_decisions_state(
|
||||
engine_client: EngineClient,
|
||||
state: State,
|
||||
args: Namespace,
|
||||
request_logger: RequestLogger | None,
|
||||
supported_tasks: tuple[SupportedTask, ...],
|
||||
default_chat_template_kwargs: dict[str, Any],
|
||||
):
|
||||
strategy = init_decisions_state(
|
||||
engine_client,
|
||||
state,
|
||||
args,
|
||||
request_logger,
|
||||
load_chat_template(args.chat_template),
|
||||
default_chat_template_kwargs,
|
||||
supported_tasks,
|
||||
)
|
||||
state.serving_structured_decisions = (
|
||||
ServingStructuredDecisions(
|
||||
state.openai_serving_models, strategy, request_logger=request_logger
|
||||
)
|
||||
if strategy is not None
|
||||
else None
|
||||
)
|
||||
@@ -1,83 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Request and response models of the structured decisions API (/v1/systemone)."""
|
||||
|
||||
import time
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import Field, model_validator
|
||||
|
||||
from vllm.entrypoints.generate.base.protocol import validate_cache_salt
|
||||
from vllm.entrypoints.serve.engine.protocol import OpenAIBaseModel
|
||||
from vllm.utils import random_uuid
|
||||
|
||||
|
||||
class QuestionSpec(OpenAIBaseModel):
|
||||
type: str
|
||||
instructions: Any = ""
|
||||
criteria: Any = None
|
||||
|
||||
|
||||
class StructuredDecisionRequest(OpenAIBaseModel):
|
||||
model: str | None = None
|
||||
state: Any = Field(..., description="What the questions are about: text or JSON.")
|
||||
questions: dict[str, QuestionSpec] = Field(
|
||||
..., description="Question id to question, asked in this order."
|
||||
)
|
||||
instructions: str | None = Field(
|
||||
default=None, description="Context placed ahead of the questions."
|
||||
)
|
||||
images: list[str] | None = Field(
|
||||
default=None, description="Image URLs or data URLs, for models that see."
|
||||
)
|
||||
audio: list[str] | None = Field(
|
||||
default=None, description="Audio URLs or data URLs, for models that hear."
|
||||
)
|
||||
videos: list[str] | None = Field(
|
||||
default=None, description="Video URLs or data URLs, for models that see."
|
||||
)
|
||||
chat_template_kwargs: dict[str, Any] | None = None
|
||||
cache_salt: str | None = Field(
|
||||
default=None,
|
||||
description=(
|
||||
"If specified, the prefix cache will be salted with the provided "
|
||||
"string to prevent an attacker to guess prompts in multi-user "
|
||||
"environments. The salt should be random, protected from "
|
||||
"access by 3rd parties, and long enough to be "
|
||||
"unpredictable (e.g., 43 characters base64-encoded, corresponding "
|
||||
"to 256 bit)."
|
||||
),
|
||||
)
|
||||
priority: int = Field(default=0, ge=-(2**63), le=2**63 - 1)
|
||||
request_id: str = Field(default_factory=random_uuid)
|
||||
|
||||
@model_validator(mode="before")
|
||||
@classmethod
|
||||
def check_cache_salt_support(cls, data: Any) -> Any:
|
||||
if isinstance(data, dict):
|
||||
validate_cache_salt(data.get("cache_salt"))
|
||||
return data
|
||||
|
||||
|
||||
class DecisionUsage(OpenAIBaseModel):
|
||||
input_tokens: int
|
||||
output_tokens: int
|
||||
|
||||
|
||||
class QuestionDiagnostics(OpenAIBaseModel):
|
||||
label_mass: float = Field(
|
||||
description="Probability the model put on the labels, over the full "
|
||||
"vocabulary. A low value means most of the probability went to tokens "
|
||||
"that are not labels."
|
||||
)
|
||||
argmax_is_label: bool
|
||||
|
||||
|
||||
class StructuredDecisionResponse(OpenAIBaseModel):
|
||||
id: str
|
||||
object: Literal["structured_decision"] = "structured_decision"
|
||||
created: int = Field(default_factory=lambda: int(time.time()))
|
||||
model: str
|
||||
answers: dict[str, dict[str, Any]]
|
||||
usage: DecisionUsage
|
||||
diagnostics: dict[str, QuestionDiagnostics]
|
||||
@@ -1,123 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
from typing import Any
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from vllm.entrypoints.openai.decisions.question_types import (
|
||||
Question,
|
||||
StructuredDecisionError,
|
||||
build_question,
|
||||
)
|
||||
from vllm.entrypoints.openai.decisions.serving import BaseServingDecisions
|
||||
from vllm.entrypoints.openai.decisions.strategies import DecisionLimits
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
|
||||
from .protocol import (
|
||||
DecisionUsage,
|
||||
QuestionDiagnostics,
|
||||
StructuredDecisionRequest,
|
||||
StructuredDecisionResponse,
|
||||
)
|
||||
|
||||
|
||||
def parse_questions(
|
||||
request: StructuredDecisionRequest,
|
||||
limits: DecisionLimits,
|
||||
) -> list[Question]:
|
||||
if not request.questions:
|
||||
raise StructuredDecisionError("questions: needs at least one question")
|
||||
if len(request.questions) > limits.max_questions:
|
||||
raise StructuredDecisionError(
|
||||
f"questions: at most {limits.max_questions} for this model"
|
||||
)
|
||||
questions = []
|
||||
for qid, spec in request.questions.items():
|
||||
if spec.model_extra:
|
||||
raise StructuredDecisionError(
|
||||
f"question {qid!r}: unknown field(s) {sorted(spec.model_extra)}"
|
||||
)
|
||||
questions.append(
|
||||
build_question(
|
||||
qid,
|
||||
spec.type,
|
||||
spec.instructions,
|
||||
spec.criteria,
|
||||
limits.max_options,
|
||||
labeled=limits.labeled,
|
||||
)
|
||||
)
|
||||
return questions
|
||||
|
||||
|
||||
class ServingStructuredDecisions(BaseServingDecisions):
|
||||
def _media(self, request: StructuredDecisionRequest) -> dict[str, list[str]]:
|
||||
urls = {
|
||||
"image": request.images,
|
||||
"audio": request.audio,
|
||||
"video": request.videos,
|
||||
}
|
||||
media = {modality: items for modality, items in urls.items() if items}
|
||||
if unsupported := media.keys() - self.strategy.modalities:
|
||||
names = ", ".join(sorted(unsupported))
|
||||
raise StructuredDecisionError(f"this model does not take {names} inputs")
|
||||
return media
|
||||
|
||||
async def create_decision(
|
||||
self,
|
||||
request: StructuredDecisionRequest,
|
||||
raw_request: Request | None = None,
|
||||
) -> StructuredDecisionResponse | ErrorResponse:
|
||||
if (error := await self._check_model(request)) is not None: # type: ignore[arg-type]
|
||||
return error
|
||||
engine_client = self.strategy.context.engine_client
|
||||
if engine_client.errored:
|
||||
raise engine_client.dead_error
|
||||
|
||||
base_id = self._base_request_id(raw_request, default=request.request_id)
|
||||
request_id = f"decision-{base_id}"
|
||||
try:
|
||||
questions = parse_questions(request, self.limits)
|
||||
media = self._media(request)
|
||||
lora_request = self._maybe_get_adapters(request) # type: ignore[arg-type]
|
||||
engine_client.check_admission(self.strategy.num_requests(questions))
|
||||
trace_headers = (
|
||||
None
|
||||
if raw_request is None
|
||||
else await self._get_trace_headers(raw_request.headers)
|
||||
)
|
||||
reads = await self.strategy.read(
|
||||
questions,
|
||||
request.instructions,
|
||||
self.strategy.render_state(request.state),
|
||||
request_id=request_id,
|
||||
chat_template_kwargs=request.chat_template_kwargs,
|
||||
lora_request=lora_request,
|
||||
priority=request.priority,
|
||||
cache_salt=request.cache_salt,
|
||||
trace_headers=trace_headers,
|
||||
media=media,
|
||||
)
|
||||
except StructuredDecisionError as e:
|
||||
return self.create_error_response(e)
|
||||
except asyncio.CancelledError:
|
||||
return self.create_error_response("Client disconnected")
|
||||
|
||||
answers: dict[str, dict[str, Any]] = {}
|
||||
diagnostics: dict[str, QuestionDiagnostics] = {}
|
||||
for q, read in zip(questions, reads):
|
||||
answers[q.id] = q.type.answer(q, read.probs, read.label_mass)
|
||||
diagnostics[q.id] = QuestionDiagnostics(
|
||||
label_mass=read.label_mass, argmax_is_label=read.argmax_is_label
|
||||
)
|
||||
return StructuredDecisionResponse(
|
||||
id=request_id,
|
||||
model=self.models.model_name(lora_request),
|
||||
answers=answers,
|
||||
usage=DecisionUsage(
|
||||
input_tokens=sum(r.input_tokens for r in reads),
|
||||
output_tokens=sum(r.output_tokens for r in reads),
|
||||
),
|
||||
diagnostics=diagnostics,
|
||||
)
|
||||
Reference in New Issue
Block a user