[Bugfix][core] replace heartbeat with pid check (#9818)
Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
This commit is contained in:
@@ -1,7 +1,5 @@
|
||||
import pickle
|
||||
import signal
|
||||
import threading
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Iterator, List, Optional, Union
|
||||
|
||||
@@ -21,7 +19,7 @@ from vllm.engine.multiprocessing import (ENGINE_DEAD_ERROR, IPC_DATA_EXT,
|
||||
RPCStartupRequest, RPCStartupResponse,
|
||||
RPCUProfileRequest)
|
||||
# yapf: enable
|
||||
from vllm.envs import VLLM_RPC_TIMEOUT, VLLM_USE_V1
|
||||
from vllm.envs import VLLM_USE_V1
|
||||
from vllm.executor.gpu_executor import GPUExecutor
|
||||
from vllm.logger import init_logger
|
||||
from vllm.outputs import RequestOutput
|
||||
@@ -108,20 +106,6 @@ class MQLLMEngine:
|
||||
# Error state.
|
||||
self._errored_with: Optional[BaseException] = None
|
||||
|
||||
# Heartbeat thread
|
||||
self.heartbeat_thread = threading.Thread(target=self._heartbeat_loop,
|
||||
daemon=True)
|
||||
self._heartbeat_stop_event = threading.Event()
|
||||
# The heartbeat needs to be faster than what the client will wait for
|
||||
# The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
|
||||
self.heartbeat_interval_seconds = VLLM_RPC_TIMEOUT / 5000.0
|
||||
|
||||
self._last_alive_time = time.time()
|
||||
# The heartbeats can tolerate a long period of the engine chugging
|
||||
# away at a generation request.
|
||||
# The VLLM_RPC_TIMEOUT duration is in ms, and we need one in seconds
|
||||
self.last_alive_threshold = VLLM_RPC_TIMEOUT * 3.0 / 1000.0
|
||||
|
||||
@property
|
||||
def dead_error(self) -> BaseException:
|
||||
if self._errored_with is not None:
|
||||
@@ -157,8 +141,6 @@ class MQLLMEngine:
|
||||
try:
|
||||
logger.debug("Starting Startup Loop.")
|
||||
self.run_startup_loop()
|
||||
logger.debug("Starting heartbeat thread")
|
||||
self.heartbeat_thread.start()
|
||||
logger.debug("Starting Engine Loop.")
|
||||
self.run_engine_loop()
|
||||
except Exception as e:
|
||||
@@ -172,7 +154,6 @@ class MQLLMEngine:
|
||||
def cleanup(self):
|
||||
"""Cleanup zeromq state on shutdown."""
|
||||
# Closes all sockets and destroys context.
|
||||
self._heartbeat_stop_event.set()
|
||||
self.ctx.destroy(linger=0)
|
||||
del self.engine
|
||||
|
||||
@@ -211,11 +192,12 @@ class MQLLMEngine:
|
||||
"""Core busy loop of the LLMEngine."""
|
||||
|
||||
while True:
|
||||
self._alive()
|
||||
if not self.engine.has_unfinished_requests():
|
||||
# Poll until there is work to do.
|
||||
while self.input_socket.poll(timeout=POLLING_TIMEOUT_MS) == 0:
|
||||
self._alive()
|
||||
# When there's no work, check on engine health and send
|
||||
# health status back to client
|
||||
self._health_check()
|
||||
self.engine.do_log_stats()
|
||||
logger.debug("Waiting for new requests in engine loop.")
|
||||
|
||||
@@ -314,32 +296,16 @@ class MQLLMEngine:
|
||||
if self.log_requests:
|
||||
logger.info("Aborted request %s.", request.request_id)
|
||||
|
||||
def _heartbeat_loop(self):
|
||||
while not self._heartbeat_stop_event.wait(
|
||||
timeout=self.heartbeat_interval_seconds):
|
||||
# Loops until the stop event is set
|
||||
self._heartbeat()
|
||||
|
||||
logger.debug("Exiting MQLLMEngine heartbeat thread")
|
||||
|
||||
def _heartbeat(self):
|
||||
def _health_check(self):
|
||||
# Send unhealthy if engine has already errored
|
||||
if self._errored_with is not None:
|
||||
self._send_unhealthy(self._errored_with)
|
||||
|
||||
# Check for life of the main loop
|
||||
elif time.time() - self._last_alive_time > self.last_alive_threshold:
|
||||
self._send_unhealthy(RuntimeError("Engine loop has died"))
|
||||
|
||||
else:
|
||||
# Otherwise- check health of the engine
|
||||
# self.engine.check_health() raises on unhealthy
|
||||
try:
|
||||
self.engine.check_health()
|
||||
self._send_healthy()
|
||||
except Exception as e:
|
||||
self._set_errored(e)
|
||||
self._send_unhealthy(e)
|
||||
try:
|
||||
self.engine.check_health()
|
||||
self._send_healthy()
|
||||
except Exception as e:
|
||||
self._set_errored(e)
|
||||
self._send_unhealthy(e)
|
||||
|
||||
def _send_outputs(self, outputs: REQUEST_OUTPUTS_T):
|
||||
"""Send List of RequestOutput to RPCClient."""
|
||||
@@ -369,9 +335,6 @@ class MQLLMEngine:
|
||||
if self._errored_with is None:
|
||||
self._errored_with = e
|
||||
|
||||
def _alive(self):
|
||||
self._last_alive_time = time.time()
|
||||
|
||||
def start_profile(self) -> None:
|
||||
if type(self.engine.model_executor) is GPUExecutor:
|
||||
self.engine.model_executor.start_profile()
|
||||
|
||||
Reference in New Issue
Block a user