[Frontend] Disable uvicorn signal handlers instead of racing them (#50916)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
@@ -312,7 +312,7 @@ async def test_shutdown_if_supervisor_server_error_on_startup(
|
|||||||
await self._shutdown_event.wait()
|
await self._shutdown_event.wait()
|
||||||
|
|
||||||
monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())
|
monkeypatch.setattr(dp_sup.asyncio, "get_running_loop", lambda: FakeLoop())
|
||||||
monkeypatch.setattr(dp_sup.uvicorn, "Server", FakeServer)
|
monkeypatch.setattr(dp_sup, "NoSignalServer", FakeServer)
|
||||||
monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)
|
monkeypatch.setattr(DPSupervisor, "_shutdown_children", fake_shutdown_children)
|
||||||
monkeypatch.setattr(DPSupervisor, "_start_children", fake_start_children)
|
monkeypatch.setattr(DPSupervisor, "_start_children", fake_start_children)
|
||||||
monkeypatch.setattr(DPSupervisor, "_monitor_children", fake_monitor_children)
|
monkeypatch.setattr(DPSupervisor, "_monitor_children", fake_monitor_children)
|
||||||
|
|||||||
@@ -2,8 +2,10 @@
|
|||||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||||
|
|
||||||
import asyncio
|
import asyncio
|
||||||
|
import contextlib
|
||||||
import signal
|
import signal
|
||||||
import socket
|
import socket
|
||||||
|
from collections.abc import Generator
|
||||||
from functools import partial
|
from functools import partial
|
||||||
from typing import Any
|
from typing import Any
|
||||||
|
|
||||||
@@ -23,6 +25,18 @@ from vllm.utils.network_utils import find_process_using_port
|
|||||||
logger = init_logger(__name__)
|
logger = init_logger(__name__)
|
||||||
|
|
||||||
|
|
||||||
|
class NoSignalServer(uvicorn.Server):
|
||||||
|
"""Uvicorn server that never installs its own SIGINT/SIGTERM handlers.
|
||||||
|
|
||||||
|
Callers register their own handlers on the event loop for graceful
|
||||||
|
shutdown; uvicorn's would race with and override them (see #49668).
|
||||||
|
"""
|
||||||
|
|
||||||
|
@contextlib.contextmanager
|
||||||
|
def capture_signals(self) -> Generator[None, None, None]:
|
||||||
|
yield
|
||||||
|
|
||||||
|
|
||||||
async def serve_http(
|
async def serve_http(
|
||||||
app: FastAPI,
|
app: FastAPI,
|
||||||
sock: socket.socket | None,
|
sock: socket.socket | None,
|
||||||
@@ -73,7 +87,7 @@ async def serve_http(
|
|||||||
config.h11_max_incomplete_event_size = h11_max_incomplete_event_size
|
config.h11_max_incomplete_event_size = h11_max_incomplete_event_size
|
||||||
config.h11_max_header_count = h11_max_header_count
|
config.h11_max_header_count = h11_max_header_count
|
||||||
config.load()
|
config.load()
|
||||||
server = uvicorn.Server(config)
|
server = NoSignalServer(config)
|
||||||
app.state.server = server
|
app.state.server = server
|
||||||
|
|
||||||
loop = asyncio.get_running_loop()
|
loop = asyncio.get_running_loop()
|
||||||
@@ -92,18 +106,6 @@ async def serve_http(
|
|||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
|
||||||
# make sure uvicorn server signal handler has been registered.
|
|
||||||
# Otherwise the app signal handler below will be overwritten in server.serve.
|
|
||||||
# server.started is set to True after server complete server.startup.
|
|
||||||
# We need to constraint the time sequence.
|
|
||||||
logger.info("API server: waiting for HTTP server to start")
|
|
||||||
while not server.started and not server_task.done():
|
|
||||||
await asyncio.sleep(0.1)
|
|
||||||
if server_task.done():
|
|
||||||
# Propagate startup failure (e.g. port bind error) instead of hanging.
|
|
||||||
await server_task
|
|
||||||
logger.info("API server: HTTP server started")
|
|
||||||
|
|
||||||
shutdown_event = asyncio.Event()
|
shutdown_event = asyncio.Event()
|
||||||
|
|
||||||
def signal_handler() -> None:
|
def signal_handler() -> None:
|
||||||
|
|||||||
@@ -23,6 +23,7 @@ import uvloop
|
|||||||
from fastapi import FastAPI, Response
|
from fastapi import FastAPI, Response
|
||||||
|
|
||||||
import vllm.envs as envs
|
import vllm.envs as envs
|
||||||
|
from vllm.entrypoints.launcher import NoSignalServer
|
||||||
from vllm.logger import init_logger
|
from vllm.logger import init_logger
|
||||||
from vllm.utils.system_utils import (
|
from vllm.utils.system_utils import (
|
||||||
decorate_logs,
|
decorate_logs,
|
||||||
@@ -335,7 +336,7 @@ class DPSupervisor:
|
|||||||
ssl_cert_reqs=self.args.ssl_cert_reqs,
|
ssl_cert_reqs=self.args.ssl_cert_reqs,
|
||||||
ssl_ciphers=self.args.ssl_ciphers,
|
ssl_ciphers=self.args.ssl_ciphers,
|
||||||
)
|
)
|
||||||
supervisor_server = uvicorn.Server(config)
|
supervisor_server = NoSignalServer(config)
|
||||||
supervisor_server_task = asyncio.create_task(
|
supervisor_server_task = asyncio.create_task(
|
||||||
supervisor_server.serve(),
|
supervisor_server.serve(),
|
||||||
name="dp-supervisor",
|
name="dp-supervisor",
|
||||||
|
|||||||
Reference in New Issue
Block a user