[Mini-VLLM] Remove the ROCm device path, debug routes and arch-gated tests
- Delete csrc/rocm, hipify, the _rocm_C extension and the setup.py rocm target path; drop the rocm requirement files - Delete the aiter, rdna3, skinny-gemm and ROCm fused-MoE kernel modules and retarget the linear kernel registries away from them - Reduce platforms/rocm.py to the GCN architecture predicates that live code still imports, and delete the RocmPlatform class and plugin - Drop the VLLM_SERVER_DEV_MODE debug surface: reset, collective_rpc, server_info and sleep routes plus the log_error_stack flag - Delete lm-format-enforcer, lark and otel-semantic-conventions-ai pins; the live server process maps none of them - Delete tests that only build architectures this fork no longer registers, and test directories left without any test file - Restore the tests/v1/kv_connector mock helpers that scheduler tests import at module level Co-authored-by: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -4,7 +4,6 @@ from argparse import Namespace
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.config import ModelConfig
|
||||
from vllm.tasks import SupportedTask
|
||||
|
||||
@@ -25,11 +24,6 @@ def register_api_routers(
|
||||
|
||||
register_models_api_router(app)
|
||||
|
||||
if envs.VLLM_SERVER_DEV_MODE:
|
||||
from vllm.entrypoints.serve import register_vllm_dev_api_routers
|
||||
|
||||
register_vllm_dev_api_routers(app)
|
||||
|
||||
if "generate" in supported_tasks:
|
||||
from vllm.entrypoints.generate.api_router import (
|
||||
register_generate_api_routers,
|
||||
|
||||
@@ -11,7 +11,6 @@ from collections.abc import Sequence
|
||||
from dataclasses import field
|
||||
from typing import Any, Literal
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.config import config
|
||||
from vllm.engine.arg_utils import AsyncEngineArgs, optional_type
|
||||
from vllm.entrypoints.openai.models.protocol import LoRAModulePath
|
||||
@@ -169,7 +168,7 @@ class BaseFrontendArgs:
|
||||
"""If set to False, output deltas will not be logged. Relevant only if
|
||||
--enable-log-outputs is set.
|
||||
"""
|
||||
log_error_stack: bool = envs.VLLM_SERVER_DEV_MODE
|
||||
log_error_stack: bool = False
|
||||
"""If set to True, log the stack trace of error responses"""
|
||||
tokens_only: bool = False
|
||||
"""
|
||||
|
||||
@@ -3,16 +3,11 @@
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .exception_handling.error_response import create_error_response
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
__all__ = [
|
||||
"create_error_response",
|
||||
"register_vllm_serve_api_routers",
|
||||
"register_vllm_dev_api_routers",
|
||||
]
|
||||
|
||||
|
||||
@@ -39,27 +34,3 @@ def register_vllm_serve_api_routers(app: FastAPI):
|
||||
|
||||
attach_tokenize_router(app)
|
||||
|
||||
|
||||
def register_vllm_dev_api_routers(app: FastAPI):
|
||||
logger.warning(
|
||||
"SECURITY WARNING: Development endpoints are enabled! "
|
||||
"This should NOT be used in production!"
|
||||
)
|
||||
|
||||
from .dev.cache.api_router import attach_router as attach_cache_router
|
||||
|
||||
attach_cache_router(app)
|
||||
|
||||
from .dev.rpc.api_router import attach_router as attach_rpc_router
|
||||
|
||||
attach_rpc_router(app)
|
||||
|
||||
from .dev.server_info.api_router import (
|
||||
attach_router as attach_server_info_router,
|
||||
)
|
||||
|
||||
attach_server_info_router(app)
|
||||
|
||||
from .dev.sleep.api_router import attach_router as attach_sleep_router
|
||||
|
||||
attach_sleep_router(app)
|
||||
|
||||
-68
@@ -1,68 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from fastapi import APIRouter, FastAPI, Query, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
@router.post("/reset_prefix_cache")
|
||||
async def reset_prefix_cache(
|
||||
raw_request: Request,
|
||||
reset_running_requests: bool = Query(default=False),
|
||||
reset_external: bool = Query(default=False),
|
||||
):
|
||||
"""Reset the local prefix cache.
|
||||
|
||||
Optionally, if the query parameter `reset_external=true`
|
||||
also resets the external (connector-managed) prefix cache.
|
||||
|
||||
Returns `{"success": bool}`. The reset fails (`success=false`) while
|
||||
blocks are still held, e.g. by running requests or in-flight async KV
|
||||
offload transfers; callers may retry.
|
||||
|
||||
Example:
|
||||
POST /reset_prefix_cache?reset_external=true
|
||||
|
||||
"""
|
||||
logger.info("Resetting prefix cache...")
|
||||
|
||||
success = await engine_client(raw_request).reset_prefix_cache(
|
||||
reset_running_requests, reset_external
|
||||
)
|
||||
return JSONResponse(content={"success": bool(success)})
|
||||
|
||||
|
||||
@router.post("/reset_mm_cache")
|
||||
async def reset_mm_cache(raw_request: Request):
|
||||
"""Reset the multi-modal cache. Note that we currently do not check if the
|
||||
multi-modal cache is successfully reset in the API server.
|
||||
"""
|
||||
logger.info("Resetting multi-modal cache...")
|
||||
await engine_client(raw_request).reset_mm_cache()
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
@router.post("/reset_encoder_cache")
|
||||
async def reset_encoder_cache(raw_request: Request):
|
||||
"""Reset the encoder cache. Note that we currently do not check if the
|
||||
encoder cache is successfully reset in the API server.
|
||||
"""
|
||||
logger.info("Resetting encoder cache...")
|
||||
await engine_client(raw_request).reset_encoder_cache()
|
||||
return Response(status_code=200)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, FastAPI, HTTPException, Request
|
||||
from fastapi.responses import JSONResponse, Response
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
@router.post("/collective_rpc")
|
||||
async def collective_rpc(raw_request: Request):
|
||||
try:
|
||||
body = await raw_request.json()
|
||||
except json.JSONDecodeError as e:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail=f"JSON decode error: {e}",
|
||||
) from e
|
||||
if not isinstance(body, dict):
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Request body must be a JSON object",
|
||||
)
|
||||
method = body.get("method")
|
||||
if method is None:
|
||||
raise HTTPException(
|
||||
status_code=HTTPStatus.BAD_REQUEST.value,
|
||||
detail="Missing 'method' in request body",
|
||||
)
|
||||
# For security reason, only serialized string args/kwargs are passed.
|
||||
# User-defined `method` is responsible for deserialization if needed.
|
||||
args: list[str] = body.get("args", [])
|
||||
kwargs: dict[str, str] = body.get("kwargs", {})
|
||||
timeout: float | None = body.get("timeout")
|
||||
results = await engine_client(raw_request).collective_rpc(
|
||||
method=method, timeout=timeout, args=tuple(args), kwargs=kwargs
|
||||
)
|
||||
if results is None:
|
||||
return Response(status_code=200)
|
||||
response: list[Any] = []
|
||||
for result in results:
|
||||
if result is None or isinstance(result, dict | list):
|
||||
response.append(result)
|
||||
else:
|
||||
response.append(str(result))
|
||||
return JSONResponse(content={"results": response})
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,63 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
import asyncio
|
||||
import functools
|
||||
from typing import Annotated, Literal
|
||||
|
||||
import pydantic
|
||||
from fastapi import APIRouter, FastAPI, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
import vllm.envs as envs
|
||||
from vllm.collect_env import get_env_info
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
PydanticVllmConfig = pydantic.TypeAdapter(VllmConfig)
|
||||
|
||||
|
||||
def _get_vllm_env_vars():
|
||||
from vllm.config.utils import normalize_value
|
||||
|
||||
vllm_envs = {}
|
||||
for key in dir(envs):
|
||||
if key.startswith("VLLM_") and "KEY" not in key:
|
||||
value = getattr(envs, key, None)
|
||||
if value is not None:
|
||||
value = normalize_value(value)
|
||||
vllm_envs[key] = value
|
||||
return vllm_envs
|
||||
|
||||
|
||||
@functools.lru_cache(maxsize=1)
|
||||
def _get_system_env_info_cached():
|
||||
return get_env_info()._asdict()
|
||||
|
||||
|
||||
@router.get("/server_info")
|
||||
async def show_server_info(
|
||||
raw_request: Request,
|
||||
config_format: Annotated[Literal["text", "json"], Query()] = "text",
|
||||
):
|
||||
vllm_config: VllmConfig = raw_request.app.state.vllm_config
|
||||
server_info = {
|
||||
"vllm_config": (
|
||||
str(vllm_config)
|
||||
if config_format == "text"
|
||||
else PydanticVllmConfig.dump_python(vllm_config, mode="json", fallback=str)
|
||||
),
|
||||
# fallback=str is needed to handle e.g. torch.dtype
|
||||
"vllm_env": _get_vllm_env_vars(),
|
||||
"system_env": await asyncio.to_thread(_get_system_env_info_cached),
|
||||
}
|
||||
return JSONResponse(content=server_info)
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,65 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, FastAPI, Query, Request
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.v1.engine import PauseMode
|
||||
|
||||
from .metrics import sleep_mode_operation_metrics
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def engine_client(request: Request) -> EngineClient:
|
||||
return request.app.state.engine_client
|
||||
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.post("/sleep")
|
||||
async def sleep(
|
||||
raw_request: Request,
|
||||
level: Annotated[int, Query(ge=0, le=2)] = 1,
|
||||
mode: Annotated[PauseMode, Query()] = "abort",
|
||||
) -> JSONResponse:
|
||||
with sleep_mode_operation_metrics().record("sleep"):
|
||||
await engine_client(raw_request).sleep(level, mode)
|
||||
return JSONResponse(content={"status": "sleeping", "level": level})
|
||||
|
||||
|
||||
@router.post("/release_kv_cache_memory")
|
||||
async def release_kv_cache_memory(raw_request: Request) -> JSONResponse:
|
||||
with sleep_mode_operation_metrics().record("release_kv_cache_memory"):
|
||||
await engine_client(raw_request).release_kv_cache_memory()
|
||||
return JSONResponse(content={"status": "kv_cache_released"})
|
||||
|
||||
|
||||
@router.post("/wake_up")
|
||||
async def wake_up(raw_request: Request) -> JSONResponse:
|
||||
tags = raw_request.query_params.getlist("tags")
|
||||
if tags == []:
|
||||
# set to None to wake up all tags if no tags are provided
|
||||
tags = None
|
||||
logger.info("wake up the engine with tags: %s", tags)
|
||||
with sleep_mode_operation_metrics().record("wake"):
|
||||
fully_awake = await engine_client(raw_request).wake_up(tags)
|
||||
return JSONResponse(
|
||||
content={"status": "awake" if fully_awake else "sleeping", "tags": tags}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/is_sleeping")
|
||||
async def is_sleeping(raw_request: Request):
|
||||
is_sleeping = await engine_client(raw_request).is_sleeping()
|
||||
return JSONResponse(content={"is_sleeping": is_sleeping})
|
||||
|
||||
|
||||
def attach_router(app: FastAPI):
|
||||
app.include_router(router)
|
||||
@@ -1,57 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""Metrics for dispatched sleep-mode frontend operations."""
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from threading import Lock
|
||||
from time import perf_counter
|
||||
from typing import Literal
|
||||
|
||||
from prometheus_client import REGISTRY, CollectorRegistry, Gauge, Histogram
|
||||
|
||||
Operation = Literal["sleep", "release_kv_cache_memory", "wake"]
|
||||
BUCKETS = (0.01, 0.1, 1, 10, 30, 60, 120, 300, 600)
|
||||
|
||||
|
||||
class SleepModeOperationMetrics:
|
||||
def __init__(self, registry: CollectorRegistry | None = None):
|
||||
registry = REGISTRY if registry is None else registry
|
||||
self.duration = Histogram(
|
||||
"vllm:sleep_mode_operation_duration_seconds",
|
||||
"Duration of one sleep-mode operation.",
|
||||
["operation"],
|
||||
buckets=BUCKETS,
|
||||
registry=registry,
|
||||
)
|
||||
self.in_flight = Gauge(
|
||||
"vllm:sleep_mode_operations_in_flight",
|
||||
"Sleep-mode operations currently awaited.",
|
||||
["operation"],
|
||||
multiprocess_mode="livesum",
|
||||
registry=registry,
|
||||
)
|
||||
|
||||
@contextmanager
|
||||
def record(self, operation: Operation) -> Iterator[None]:
|
||||
active = self.in_flight.labels(operation)
|
||||
started = perf_counter()
|
||||
active.inc()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
active.dec()
|
||||
self.duration.labels(operation).observe(perf_counter() - started)
|
||||
|
||||
|
||||
_metrics: SleepModeOperationMetrics | None = None
|
||||
_metrics_lock = Lock()
|
||||
|
||||
|
||||
def sleep_mode_operation_metrics() -> SleepModeOperationMetrics:
|
||||
global _metrics
|
||||
if _metrics is None:
|
||||
with _metrics_lock:
|
||||
if _metrics is None:
|
||||
_metrics = SleepModeOperationMetrics()
|
||||
return _metrics
|
||||
Reference in New Issue
Block a user