[Bugfix] Fix MQLLMEngine hanging (#9973)
Signed-off-by: rshaw@neuralmagic.com <rshaw@neuralmagic.com>
This commit is contained in:
@@ -112,7 +112,11 @@ class MQLLMEngineClient(EngineClient):
|
||||
|
||||
# Stream for each individual request.
|
||||
self.output_queues: Dict[str, asyncio.Queue] = {}
|
||||
self.output_loop = asyncio.create_task(self.run_output_handler_loop())
|
||||
|
||||
# Loop to handle output of the LLMEngine periodically.
|
||||
# Started after the MQLLMEngine is ready so that we can
|
||||
# build the Client in an executor to enable clean shutdown.
|
||||
self.output_loop: Optional[asyncio.Task] = None
|
||||
|
||||
# Loop to check health of the LLMEngine periodically.
|
||||
# Started after the MQLLMEngine is ready.
|
||||
@@ -247,6 +251,9 @@ class MQLLMEngineClient(EngineClient):
|
||||
async def setup(self):
|
||||
"""Setup the client before it starts sending server requests."""
|
||||
|
||||
# Start output_loop
|
||||
self.output_loop = asyncio.create_task(self.run_output_handler_loop())
|
||||
|
||||
with self.get_data_socket() as socket:
|
||||
# Wait until server is ready.
|
||||
response = await self._wait_for_server_rpc(socket)
|
||||
@@ -265,7 +272,8 @@ class MQLLMEngineClient(EngineClient):
|
||||
# Cancel background tasks.
|
||||
if self.health_loop is not None:
|
||||
self.health_loop.cancel()
|
||||
self.output_loop.cancel()
|
||||
if self.output_loop is not None:
|
||||
self.output_loop.cancel()
|
||||
|
||||
def _set_errored(self, e: BaseException):
|
||||
logger.exception(repr(e))
|
||||
|
||||
@@ -349,16 +349,22 @@ class MQLLMEngine:
|
||||
self.engine.model_executor._run_workers("stop_profile")
|
||||
|
||||
|
||||
def signal_handler(*_) -> None:
|
||||
raise KeyboardInterrupt("MQLLMEngine terminated")
|
||||
|
||||
|
||||
def run_mp_engine(engine_args: AsyncEngineArgs, usage_context: UsageContext,
|
||||
ipc_path: str):
|
||||
ipc_path: str, engine_alive):
|
||||
try:
|
||||
engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
|
||||
usage_context=usage_context,
|
||||
ipc_path=ipc_path)
|
||||
|
||||
def signal_handler(*_) -> None:
|
||||
# Interrupt server on sigterm
|
||||
raise KeyboardInterrupt("MQLLMEngine terminated")
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
engine.start()
|
||||
|
||||
engine = MQLLMEngine.from_engine_args(engine_args=engine_args,
|
||||
usage_context=usage_context,
|
||||
ipc_path=ipc_path)
|
||||
engine.start()
|
||||
except BaseException as e:
|
||||
logger.exception(e)
|
||||
engine_alive.value = False
|
||||
raise e
|
||||
|
||||
Reference in New Issue
Block a user