[Frontend] Move api_server.py out openai folder (#52131)
Signed-off-by: wang.yuqi <yuqi.wang@daocloud.io>
This commit is contained in:
@@ -1,12 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Shared constants for vLLM entrypoints.
|
||||
"""
|
||||
|
||||
# HTTP header limits for h11 parser
|
||||
# These constants help mitigate header abuse attacks
|
||||
H11_MAX_INCOMPLETE_EVENT_SIZE_DEFAULT = 4194304 # 4 MB
|
||||
H11_MAX_HEADER_COUNT_DEFAULT = 256
|
||||
|
||||
MCP_PREFIX = "mcp_"
|
||||
@@ -1,101 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import asyncio
|
||||
import json
|
||||
from argparse import Namespace
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from vllm import envs
|
||||
from vllm.engine.protocol import EngineClient
|
||||
from vllm.logger import init_logger
|
||||
from vllm.utils.gc_utils import freeze_gc_heap
|
||||
|
||||
logger = init_logger("vllm.entrypoints.openai.server_utils")
|
||||
|
||||
|
||||
def load_log_config(log_config_file: str | None) -> dict | None:
|
||||
if not log_config_file:
|
||||
return None
|
||||
try:
|
||||
with open(log_config_file) as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.warning(
|
||||
"Failed to load log config from file %s: error %s", log_config_file, e
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def get_uvicorn_log_config(args: Namespace) -> dict | None:
|
||||
"""
|
||||
Get the uvicorn log config based on the provided arguments.
|
||||
|
||||
Priority:
|
||||
1. If log_config_file is specified, use it
|
||||
2. If disable_access_log_for_endpoints is specified, create a config with
|
||||
the access log filter
|
||||
3. Otherwise, return None (use uvicorn defaults)
|
||||
"""
|
||||
# First, try to load from file if specified
|
||||
log_config = load_log_config(args.log_config_file)
|
||||
if log_config is not None:
|
||||
return log_config
|
||||
|
||||
# If endpoints to filter are specified, create a config with the filter
|
||||
if args.disable_access_log_for_endpoints:
|
||||
from vllm.logging_utils import create_uvicorn_log_config
|
||||
|
||||
# Parse comma-separated string into list
|
||||
excluded_paths = [
|
||||
p.strip()
|
||||
for p in args.disable_access_log_for_endpoints.split(",")
|
||||
if p.strip()
|
||||
]
|
||||
return create_uvicorn_log_config(
|
||||
excluded_paths=excluded_paths,
|
||||
log_level=args.uvicorn_log_level,
|
||||
)
|
||||
|
||||
return None
|
||||
|
||||
|
||||
_running_tasks: set[asyncio.Task] = set()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
try:
|
||||
if app.state.log_stats:
|
||||
engine_client: EngineClient = app.state.engine_client
|
||||
|
||||
async def _force_log():
|
||||
while True:
|
||||
await asyncio.sleep(envs.VLLM_LOG_STATS_INTERVAL)
|
||||
await engine_client.do_log_stats()
|
||||
|
||||
task = asyncio.create_task(_force_log())
|
||||
_running_tasks.add(task)
|
||||
task.add_done_callback(_running_tasks.remove)
|
||||
else:
|
||||
task = None
|
||||
|
||||
# Mark the startup heap as static so that it's ignored by GC.
|
||||
# Reduces pause times of oldest generation collections.
|
||||
freeze_gc_heap()
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
if task is not None:
|
||||
task.cancel()
|
||||
for attr_name in (
|
||||
"openai_serving_transcription",
|
||||
"openai_serving_translation",
|
||||
):
|
||||
serving = getattr(app.state, attr_name, None)
|
||||
if serving is not None and hasattr(serving, "shutdown"):
|
||||
serving.shutdown()
|
||||
finally:
|
||||
# Ensure app state including engine ref is gc'd
|
||||
del app.state
|
||||
@@ -1,78 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
import asyncio
|
||||
from collections.abc import Callable
|
||||
from ssl import SSLContext
|
||||
|
||||
from watchfiles import Change, awatch
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SSLCertRefresher:
|
||||
"""A class that monitors SSL certificate files and
|
||||
reloads them when they change.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
ssl_context: SSLContext,
|
||||
key_path: str | None = None,
|
||||
cert_path: str | None = None,
|
||||
ca_path: str | None = None,
|
||||
) -> None:
|
||||
self.ssl = ssl_context
|
||||
self.key_path = key_path
|
||||
self.cert_path = cert_path
|
||||
self.ca_path = ca_path
|
||||
|
||||
# Setup certification chain watcher
|
||||
def update_ssl_cert_chain(change: Change, file_path: str) -> None:
|
||||
logger.info("Reloading SSL certificate chain")
|
||||
assert self.key_path and self.cert_path
|
||||
self.ssl.load_cert_chain(self.cert_path, self.key_path)
|
||||
|
||||
self.watch_ssl_cert_task = None
|
||||
if self.key_path and self.cert_path:
|
||||
self.watch_ssl_cert_task = asyncio.create_task(
|
||||
self._watch_files(
|
||||
[self.key_path, self.cert_path], update_ssl_cert_chain
|
||||
)
|
||||
)
|
||||
|
||||
# Setup CA files watcher
|
||||
def update_ssl_ca(change: Change, file_path: str) -> None:
|
||||
logger.info("Reloading SSL CA certificates")
|
||||
assert self.ca_path
|
||||
self.ssl.load_verify_locations(self.ca_path)
|
||||
|
||||
self.watch_ssl_ca_task = None
|
||||
if self.ca_path:
|
||||
self.watch_ssl_ca_task = asyncio.create_task(
|
||||
self._watch_files([self.ca_path], update_ssl_ca)
|
||||
)
|
||||
|
||||
async def _watch_files(self, paths, fun: Callable[[Change, str], None]) -> None:
|
||||
"""Watch multiple file paths asynchronously."""
|
||||
logger.info("SSLCertRefresher monitors files: %s", paths)
|
||||
async for changes in awatch(*paths):
|
||||
try:
|
||||
for change, file_path in changes:
|
||||
logger.info("File change detected: %s - %s", change.name, file_path)
|
||||
fun(change, file_path)
|
||||
except Exception as e:
|
||||
logger.error(
|
||||
"SSLCertRefresher failed taking action on file change. Error: %s", e
|
||||
)
|
||||
|
||||
def stop(self) -> None:
|
||||
"""Stop watching files."""
|
||||
if self.watch_ssl_cert_task:
|
||||
self.watch_ssl_cert_task.cancel()
|
||||
self.watch_ssl_cert_task = None
|
||||
if self.watch_ssl_ca_task:
|
||||
self.watch_ssl_ca_task.cancel()
|
||||
self.watch_ssl_ca_task = None
|
||||
Reference in New Issue
Block a user