[Mini-VLLM] Cut non-OpenAI surfaces, dead kernels, Mistral stack
- Remove Anthropic, Cohere, SageMaker, decisions, structured-decisions, rl, render-launcher and batch entrypoints; keep scale-out token routes - Drop the Swagger static bundle and the /docs, /redoc routes - Delete device-keyed tuning configs for fused_moe, quant utils, mamba select-state and helion; no sm86 entry ever existed for any of them - Remove ten unused rope variants plus the fbgemm_fp8 and fp_quant quantizers and their config, metrics and loader plumbing - Cut the CPU device build: csrc/cpu, cpu_extension.cmake, cpu requirements and the setup.py cpu target path - Delete 29 environment variables nothing reads and 14 chat-template fallbacks for models this fork no longer registers - Remove nine dead CUDA ops (deepseek_v4, kimi_k3, minimax, muse_spark, dsv3 gemm) with their kernels, bindings, wrappers and CMake entries - Delete the connector test suites that targeted removed connectors - Cut the Mistral tokenizer and parser stack, its tests, and the mistral_common and ijson dependency pins - Restore the Qwen4Exp registry rows and nvidia/mtp.py that an earlier cut deleted while keeping the model tree Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -1,118 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.distributed.elastic_ep.external_elastic_ep import ExternalElasticEPScalePhase
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.entrypoints.serve.elastic_ep.middleware import get_scaling_elastic_ep
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post(
|
||||
"/scale_elastic_ep",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.OK.value: {"model": dict},
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.REQUEST_TIMEOUT.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
async def scale_elastic_ep(raw_request: Request):
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(status_code=400, detail="Invalid JSON format") from e
|
||||
|
||||
new_data_parallel_size = body.get("new_data_parallel_size")
|
||||
drain_timeout = body.get("drain_timeout", 120) # Default 2 minutes
|
||||
|
||||
if new_data_parallel_size is None:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="new_data_parallel_size is required"
|
||||
)
|
||||
|
||||
if not isinstance(new_data_parallel_size, int) or new_data_parallel_size <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="new_data_parallel_size must be a positive integer",
|
||||
)
|
||||
|
||||
if not isinstance(drain_timeout, int) or drain_timeout <= 0:
|
||||
raise HTTPException(
|
||||
status_code=400, detail="drain_timeout must be a positive integer"
|
||||
)
|
||||
|
||||
client = engine_client(raw_request)
|
||||
try:
|
||||
await client.scale_elastic_ep(new_data_parallel_size, drain_timeout)
|
||||
return JSONResponse(
|
||||
{
|
||||
"message": f"Scaled to {new_data_parallel_size} data parallel engines",
|
||||
}
|
||||
)
|
||||
except TimeoutError as e:
|
||||
raise HTTPException(
|
||||
status_code=408,
|
||||
detail="Scale failed due to request drain timeout "
|
||||
f"after {drain_timeout} seconds",
|
||||
) from e
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e)) from e
|
||||
except Exception as e:
|
||||
logger.error("Scale failed: %s", e)
|
||||
raise HTTPException(status_code=500, detail="Scale failed") from e
|
||||
|
||||
|
||||
@router.post("/is_scaling_elastic_ep")
|
||||
async def is_scaling_elastic_ep(raw_request: Request):
|
||||
# External operation status comes from the shared store. Middleware gating
|
||||
# remains process-local, so scaling requests must reach every old-rank API.
|
||||
try:
|
||||
status = await engine_client(raw_request).get_external_elastic_ep_status()
|
||||
except Exception as e:
|
||||
logger.warning("Failed to query external Elastic EP status: %s", e)
|
||||
raise HTTPException(
|
||||
status_code=503,
|
||||
detail="External Elastic EP status is temporarily unavailable",
|
||||
) from e
|
||||
if status is None:
|
||||
# Non-external EEP modes retain the process-local middleware state.
|
||||
is_scaling = get_scaling_elastic_ep()
|
||||
status = {
|
||||
"phase": (
|
||||
ExternalElasticEPScalePhase.COMMITTING.value
|
||||
if is_scaling
|
||||
else ExternalElasticEPScalePhase.IDLE.value
|
||||
),
|
||||
"epoch": None,
|
||||
"error": None,
|
||||
"requested_data_parallel_size": None,
|
||||
}
|
||||
else:
|
||||
is_scaling = status["phase"] in (
|
||||
ExternalElasticEPScalePhase.PREPARING.value,
|
||||
ExternalElasticEPScalePhase.COMMITTING.value,
|
||||
)
|
||||
return JSONResponse({"is_scaling_elastic_ep": is_scaling, **status})
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -16,7 +16,3 @@ def register_instrumentator_api_routers(app: FastAPI):
|
||||
from .metrics import attach_router as metrics_attach_router
|
||||
|
||||
metrics_attach_router(app)
|
||||
|
||||
from .offline_docs import attach_router as offline_docs_attach_router
|
||||
|
||||
offline_docs_attach_router(app)
|
||||
|
||||
@@ -1,50 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Offline FastAPI documentation support for air-gapped environments."""
|
||||
|
||||
import pathlib
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.openapi.docs import (
|
||||
get_swagger_ui_html,
|
||||
get_swagger_ui_oauth2_redirect_html,
|
||||
)
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI) -> None:
|
||||
"""Attach offline docs router if enabled via args."""
|
||||
args = getattr(app.state, "args", None)
|
||||
if args is None or not getattr(args, "enable_offline_docs", False):
|
||||
return
|
||||
|
||||
static_dir = pathlib.Path(__file__).parent / "static"
|
||||
|
||||
if not static_dir.exists():
|
||||
logger.warning(
|
||||
"Static directory not found at %s. Offline docs will not be available.",
|
||||
static_dir,
|
||||
)
|
||||
return
|
||||
|
||||
app.mount("/static", StaticFiles(directory=str(static_dir)), name="static")
|
||||
|
||||
@app.get("/docs", include_in_schema=False)
|
||||
async def custom_swagger_ui_html():
|
||||
return get_swagger_ui_html(
|
||||
openapi_url=app.openapi_url,
|
||||
title=app.title + " - Swagger UI",
|
||||
oauth2_redirect_url=app.swagger_ui_oauth2_redirect_url,
|
||||
swagger_js_url="/static/swagger-ui-bundle.js",
|
||||
swagger_css_url="/static/swagger-ui.css",
|
||||
)
|
||||
|
||||
@app.get(app.swagger_ui_oauth2_redirect_url, include_in_schema=False)
|
||||
async def swagger_ui_redirect():
|
||||
return get_swagger_ui_oauth2_redirect_html()
|
||||
|
||||
logger.info("Offline documentation enabled with vendored static assets")
|
||||
File diff suppressed because one or more lines are too long
File diff suppressed because one or more lines are too long
@@ -8,7 +8,7 @@ from starlette.datastructures import Headers
|
||||
from starlette.responses import JSONResponse
|
||||
from starlette.types import ASGIApp, Receive, Scope, Send
|
||||
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference", "/cohere")
|
||||
GUARDED_PREFIX = ("/v1", "/v2", "/inference")
|
||||
|
||||
|
||||
class AuthenticationMiddleware:
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
"""SageMaker-specific integration for vLLM."""
|
||||
@@ -1,156 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import json
|
||||
import logging
|
||||
from collections.abc import Awaitable, Callable
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
import pydantic
|
||||
from fastapi import APIRouter, Depends, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.entrypoints.generate.factories import get_generate_invocation_types
|
||||
from vllm.entrypoints.serve.engine.protocol import ErrorResponse
|
||||
from vllm.entrypoints.serve.engine.serving import BaseServing
|
||||
from vllm.entrypoints.serve.instrumentator.basic import base
|
||||
from vllm.entrypoints.serve.instrumentator.health import health
|
||||
from vllm.entrypoints.serve.utils.api_utils import validate_json_request
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
# TODO: RequestType = TypeForm[BaseModel] when recognized by type checkers
|
||||
# (requires typing_extensions >= 4.13)
|
||||
RequestType = Any
|
||||
GetHandlerFn = Callable[[Request], BaseServing | None]
|
||||
EndpointFn = Callable[[RequestType, Request], Awaitable[Any]]
|
||||
|
||||
|
||||
def _snapshot_handler_levels() -> list[tuple[logging.Handler, int]]:
|
||||
"""Snapshot handler levels for loggers that may be affected by third-party
|
||||
logging configuration side effects (e.g. model_hosting_container_standards
|
||||
calling configure_root_logger() at import time)."""
|
||||
loggers = [logging.getLogger(), logging.getLogger("vllm")]
|
||||
seen: set[int] = set()
|
||||
snapshot: list[tuple[logging.Handler, int]] = []
|
||||
for logger in loggers:
|
||||
for handler in logger.handlers:
|
||||
if id(handler) not in seen:
|
||||
seen.add(id(handler))
|
||||
snapshot.append((handler, handler.level))
|
||||
return snapshot
|
||||
|
||||
|
||||
def _restore_handler_levels(
|
||||
snapshot: list[tuple[logging.Handler, int]],
|
||||
) -> None:
|
||||
"""Restore handler levels from a snapshot."""
|
||||
for handler, level in snapshot:
|
||||
handler.setLevel(level)
|
||||
|
||||
|
||||
def attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Attach the SageMaker hosting endpoints to the API server.
|
||||
|
||||
Handler levels are snapshotted and restored because importing
|
||||
model_hosting_container_standards may reconfigure root logging.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
_attach_router(app, supported_tasks, model_config)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
|
||||
|
||||
def _attach_router(
|
||||
app: FastAPI,
|
||||
supported_tasks: tuple["SupportedTask", ...],
|
||||
model_config: ModelConfig | None = None,
|
||||
):
|
||||
"""Register the SageMaker hosting routes (/ping, /invocations) on the
|
||||
app."""
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# NOTE: Construct the TypeAdapters only once
|
||||
INVOCATION_TYPES = get_generate_invocation_types(supported_tasks, model_config)
|
||||
|
||||
INVOCATION_VALIDATORS = [
|
||||
(pydantic.TypeAdapter(request_type), (get_handler, endpoint))
|
||||
for request_type, (get_handler, endpoint) in INVOCATION_TYPES
|
||||
]
|
||||
|
||||
@router.post("/ping", response_class=Response)
|
||||
@router.get("/ping", response_class=Response)
|
||||
@sagemaker_standards.register_ping_handler
|
||||
async def ping(raw_request: Request) -> Response:
|
||||
"""Ping check. Endpoint required for SageMaker."""
|
||||
return await health(raw_request)
|
||||
|
||||
@router.post(
|
||||
"/invocations",
|
||||
dependencies=[Depends(validate_json_request)],
|
||||
responses={
|
||||
HTTPStatus.BAD_REQUEST.value: {"model": ErrorResponse},
|
||||
HTTPStatus.UNSUPPORTED_MEDIA_TYPE.value: {"model": ErrorResponse},
|
||||
HTTPStatus.INTERNAL_SERVER_ERROR.value: {"model": ErrorResponse},
|
||||
},
|
||||
)
|
||||
@sagemaker_standards.register_invocation_handler
|
||||
@sagemaker_standards.stateful_session_manager()
|
||||
@sagemaker_standards.inject_adapter_id(adapter_path="model")
|
||||
async def invocations(raw_request: Request):
|
||||
"""For SageMaker, routes requests based on the request type."""
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
|
||||
valid_endpoints = [
|
||||
(validator, endpoint)
|
||||
for validator, (get_handler, endpoint) in INVOCATION_VALIDATORS
|
||||
if get_handler(raw_request) is not None
|
||||
]
|
||||
|
||||
for request_validator, endpoint in valid_endpoints:
|
||||
try:
|
||||
request = request_validator.validate_python(body)
|
||||
except pydantic.ValidationError:
|
||||
continue
|
||||
|
||||
return await endpoint(request, raw_request)
|
||||
|
||||
type_names = [
|
||||
t.__name__ if isinstance(t := validator._type, type) else str(t)
|
||||
for validator, _ in valid_endpoints
|
||||
]
|
||||
msg = f"Cannot find suitable handler for request. Expected one of: {type_names}"
|
||||
res = base(raw_request).create_error_response(message=msg)
|
||||
return JSONResponse(content=res.model_dump(), status_code=res.error.code)
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
|
||||
def sagemaker_standards_bootstrap(app: FastAPI) -> FastAPI:
|
||||
"""Bootstrap the app with the SageMaker hosting standards.
|
||||
|
||||
Handler levels are restored right after the import because importing
|
||||
model_hosting_container_standards may reconfigure root logging, and
|
||||
bootstrap must run with the original levels.
|
||||
"""
|
||||
snapshot = _snapshot_handler_levels()
|
||||
try:
|
||||
import model_hosting_container_standards.sagemaker as sagemaker_standards
|
||||
|
||||
app = sagemaker_standards.bootstrap(app)
|
||||
finally:
|
||||
_restore_handler_levels(snapshot)
|
||||
return app
|
||||
Reference in New Issue
Block a user