[Frontend][Core] Re-add shutdown timeout - allowing in-flight requests to finish (#36666)
Signed-off-by: Mark McLoughlin <markmc@redhat.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.5
Nick Hill
parent
5a3f1eb62f
commit
7afe0faab1
@@ -3,6 +3,7 @@
|
||||
|
||||
import argparse
|
||||
import signal
|
||||
import time
|
||||
|
||||
import uvloop
|
||||
|
||||
@@ -222,8 +223,12 @@ def run_headless(args: argparse.Namespace):
|
||||
try:
|
||||
engine_manager.join_first()
|
||||
finally:
|
||||
timeout = None
|
||||
if shutdown_requested:
|
||||
timeout = vllm_config.shutdown_timeout
|
||||
logger.info("Waiting up to %d seconds for processes to exit", timeout)
|
||||
engine_manager.shutdown(timeout=timeout)
|
||||
logger.info("Shutting down.")
|
||||
engine_manager.close()
|
||||
|
||||
|
||||
def run_multi_api_server(args: argparse.Namespace):
|
||||
@@ -234,6 +239,19 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
if num_api_servers > 1:
|
||||
setup_multiprocess_prometheus()
|
||||
|
||||
shutdown_requested = False
|
||||
|
||||
# Catch SIGTERM and SIGINT to allow graceful shutdown.
|
||||
def signal_handler(signum, frame):
|
||||
nonlocal shutdown_requested
|
||||
logger.debug("Received %d signal.", signum)
|
||||
if not shutdown_requested:
|
||||
shutdown_requested = True
|
||||
raise SystemExit
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
listen_address, sock = setup_server(args)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
@@ -295,11 +313,29 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
api_server_manager = APIServerProcessManager(**api_server_manager_kwargs)
|
||||
|
||||
# Wait for API servers
|
||||
wait_for_completion_or_failure(
|
||||
api_server_manager=api_server_manager,
|
||||
engine_manager=local_engine_manager,
|
||||
coordinator=coordinator,
|
||||
)
|
||||
try:
|
||||
wait_for_completion_or_failure(
|
||||
api_server_manager=api_server_manager,
|
||||
engine_manager=local_engine_manager,
|
||||
coordinator=coordinator,
|
||||
)
|
||||
finally:
|
||||
timeout = shutdown_by = None
|
||||
if shutdown_requested:
|
||||
timeout = vllm_config.shutdown_timeout
|
||||
shutdown_by = time.monotonic() + timeout
|
||||
logger.info("Waiting up to %d seconds for processes to exit", timeout)
|
||||
|
||||
def to_timeout(deadline: float | None) -> float | None:
|
||||
return (
|
||||
deadline if deadline is None else max(deadline - time.monotonic(), 0.0)
|
||||
)
|
||||
|
||||
api_server_manager.shutdown(timeout=timeout)
|
||||
if local_engine_manager:
|
||||
local_engine_manager.shutdown(timeout=to_timeout(shutdown_by))
|
||||
if coordinator:
|
||||
coordinator.shutdown(timeout=to_timeout(shutdown_by))
|
||||
|
||||
|
||||
def run_api_server_worker_proc(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
import asyncio
|
||||
import signal
|
||||
import socket
|
||||
from functools import partial
|
||||
from typing import Any
|
||||
|
||||
import uvicorn
|
||||
@@ -91,12 +92,10 @@ async def serve_http(
|
||||
)
|
||||
)
|
||||
|
||||
shutdown_event = asyncio.Event()
|
||||
|
||||
def signal_handler() -> None:
|
||||
# prevents the uvicorn signal handler to exit early
|
||||
server_task.cancel()
|
||||
watchdog_task.cancel()
|
||||
if ssl_cert_refresher:
|
||||
ssl_cert_refresher.stop()
|
||||
shutdown_event.set()
|
||||
|
||||
async def dummy_shutdown() -> None:
|
||||
pass
|
||||
@@ -104,6 +103,24 @@ async def serve_http(
|
||||
loop.add_signal_handler(signal.SIGINT, signal_handler)
|
||||
loop.add_signal_handler(signal.SIGTERM, signal_handler)
|
||||
|
||||
async def handle_shutdown() -> None:
|
||||
await shutdown_event.wait()
|
||||
|
||||
engine_client = app.state.engine_client
|
||||
timeout = engine_client.vllm_config.shutdown_timeout
|
||||
|
||||
await loop.run_in_executor(
|
||||
None, partial(engine_client.shutdown, timeout=timeout)
|
||||
)
|
||||
|
||||
server.should_exit = True
|
||||
server_task.cancel()
|
||||
watchdog_task.cancel()
|
||||
if ssl_cert_refresher:
|
||||
ssl_cert_refresher.stop()
|
||||
|
||||
shutdown_task = loop.create_task(handle_shutdown())
|
||||
|
||||
try:
|
||||
await server_task
|
||||
return dummy_shutdown()
|
||||
@@ -120,6 +137,7 @@ async def serve_http(
|
||||
logger.info("Shutting down FastAPI HTTP server.")
|
||||
return server.shutdown()
|
||||
finally:
|
||||
shutdown_task.cancel()
|
||||
watchdog_task.cancel()
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user