[Frontend][Core] Re-add shutdown timeout - allowing in-flight requests to finish (#36666)

Signed-off-by: Mark McLoughlin <markmc@redhat.com>
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Mark McLoughlin
2026-03-13 12:10:06 -07:00
committed by GitHub
co-authored by Claude Sonnet 4.5 Nick Hill
parent 5a3f1eb62f
commit 7afe0faab1
14 changed files with 762 additions and 96 deletions
+12 -12
View File
@@ -128,7 +128,7 @@ class EngineCoreClient(ABC):
return AsyncMPClient(*client_args)
@abstractmethod
def shutdown(self): ...
def shutdown(self, timeout: float | None = None) -> None: ...
def get_output(self) -> EngineCoreOutputs:
raise NotImplementedError
@@ -298,7 +298,7 @@ class InprocClient(EngineCoreClient):
if len(request_ids) > 0:
self.engine_core.abort_requests(request_ids)
def shutdown(self) -> None:
def shutdown(self, timeout: float | None = None) -> None:
self.engine_core.shutdown()
def profile(self, is_start: bool = True, profile_prefix: str | None = None) -> None:
@@ -390,9 +390,9 @@ class BackgroundResources:
self.engine_dead = True
if self.engine_manager is not None:
self.engine_manager.close()
self.engine_manager.shutdown()
if self.coordinator is not None:
self.coordinator.close()
self.coordinator.shutdown()
if isinstance(self.output_socket, zmq.asyncio.Socket):
# Async case.
@@ -581,10 +581,7 @@ class MPClient(EngineCoreClient):
)
with launch_core_engines(
vllm_config,
executor_class,
log_stats,
addresses,
vllm_config, executor_class, log_stats, addresses
) as (engine_manager, coordinator, addresses):
self.resources.coordinator = coordinator
self.resources.engine_manager = engine_manager
@@ -649,9 +646,12 @@ class MPClient(EngineCoreClient):
if not success:
self._finalizer()
def shutdown(self):
# Terminate background resources.
self._finalizer()
def shutdown(self, timeout: float | None = None) -> None:
"""Shutdown engine manager under timeout and clean up resources."""
if self._finalizer.detach() is not None:
if self.resources.engine_manager is not None:
self.resources.engine_manager.shutdown(timeout=timeout)
self.resources()
def _format_exception(self, e: Exception) -> Exception:
"""If errored, use EngineDeadError so root cause is clear."""
@@ -695,7 +695,7 @@ class MPClient(EngineCoreClient):
sentinels = [proc.sentinel for proc in engine_processes]
died = multiprocessing.connection.wait(sentinels)
_self = self_ref()
if not _self or _self.resources.engine_dead:
if not _self or not _self._finalizer.alive or _self.resources.engine_dead:
return
_self.resources.engine_dead = True
proc_name = next(