[Frontend] Reapply "Factor out code for running uvicorn" (#7095)

This commit is contained in:
Cyrus Leung
2024-08-04 20:40:51 -07:00
committed by GitHub
parent 7b86e7c9cd
commit cc08fc7225
3 changed files with 124 additions and 81 deletions
+46
View File
@@ -0,0 +1,46 @@
import asyncio
import signal
from typing import Any
import uvicorn
from fastapi import FastAPI
from vllm.logger import init_logger
logger = init_logger(__name__)
async def serve_http(app: FastAPI, **uvicorn_kwargs: Any):
logger.info("Available routes are:")
for route in app.routes:
methods = getattr(route, "methods", None)
path = getattr(route, "path", None)
if methods is None or path is None:
continue
logger.info("Route: %s, Methods: %s", path, ', '.join(methods))
config = uvicorn.Config(app, **uvicorn_kwargs)
server = uvicorn.Server(config)
loop = asyncio.get_running_loop()
server_task = loop.create_task(server.serve())
def signal_handler() -> None:
# prevents the uvicorn signal handler to exit early
server_task.cancel()
async def dummy_shutdown() -> None:
pass
loop.add_signal_handler(signal.SIGINT, signal_handler)
loop.add_signal_handler(signal.SIGTERM, signal_handler)
try:
await server_task
return dummy_shutdown()
except asyncio.CancelledError:
logger.info("Gracefully stopping http server")
return server.shutdown()