[Frontend][Core] Re-add shutdown timeout - allowing in-flight requests to finish (#36666)
Signed-off-by: Mark McLoughlin <markmc@redhat.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Claude Sonnet 4.5
Nick Hill
parent
5a3f1eb62f
commit
7afe0faab1
@@ -3,6 +3,7 @@
|
||||
|
||||
import argparse
|
||||
import signal
|
||||
import time
|
||||
|
||||
import uvloop
|
||||
|
||||
@@ -222,8 +223,12 @@ def run_headless(args: argparse.Namespace):
|
||||
try:
|
||||
engine_manager.join_first()
|
||||
finally:
|
||||
timeout = None
|
||||
if shutdown_requested:
|
||||
timeout = vllm_config.shutdown_timeout
|
||||
logger.info("Waiting up to %d seconds for processes to exit", timeout)
|
||||
engine_manager.shutdown(timeout=timeout)
|
||||
logger.info("Shutting down.")
|
||||
engine_manager.close()
|
||||
|
||||
|
||||
def run_multi_api_server(args: argparse.Namespace):
|
||||
@@ -234,6 +239,19 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
if num_api_servers > 1:
|
||||
setup_multiprocess_prometheus()
|
||||
|
||||
shutdown_requested = False
|
||||
|
||||
# Catch SIGTERM and SIGINT to allow graceful shutdown.
|
||||
def signal_handler(signum, frame):
|
||||
nonlocal shutdown_requested
|
||||
logger.debug("Received %d signal.", signum)
|
||||
if not shutdown_requested:
|
||||
shutdown_requested = True
|
||||
raise SystemExit
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
|
||||
listen_address, sock = setup_server(args)
|
||||
|
||||
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
|
||||
@@ -295,11 +313,29 @@ def run_multi_api_server(args: argparse.Namespace):
|
||||
api_server_manager = APIServerProcessManager(**api_server_manager_kwargs)
|
||||
|
||||
# Wait for API servers
|
||||
wait_for_completion_or_failure(
|
||||
api_server_manager=api_server_manager,
|
||||
engine_manager=local_engine_manager,
|
||||
coordinator=coordinator,
|
||||
)
|
||||
try:
|
||||
wait_for_completion_or_failure(
|
||||
api_server_manager=api_server_manager,
|
||||
engine_manager=local_engine_manager,
|
||||
coordinator=coordinator,
|
||||
)
|
||||
finally:
|
||||
timeout = shutdown_by = None
|
||||
if shutdown_requested:
|
||||
timeout = vllm_config.shutdown_timeout
|
||||
shutdown_by = time.monotonic() + timeout
|
||||
logger.info("Waiting up to %d seconds for processes to exit", timeout)
|
||||
|
||||
def to_timeout(deadline: float | None) -> float | None:
|
||||
return (
|
||||
deadline if deadline is None else max(deadline - time.monotonic(), 0.0)
|
||||
)
|
||||
|
||||
api_server_manager.shutdown(timeout=timeout)
|
||||
if local_engine_manager:
|
||||
local_engine_manager.shutdown(timeout=to_timeout(shutdown_by))
|
||||
if coordinator:
|
||||
coordinator.shutdown(timeout=to_timeout(shutdown_by))
|
||||
|
||||
|
||||
def run_api_server_worker_proc(
|
||||
|
||||
Reference in New Issue
Block a user