[ROCm] Give EngineCore cleanup grace after request abort (#52281)
Signed-off-by: Andreas Karatzas <akaratza@amd.com> Signed-off-by: Andreas Karatzas <Andreas.Karatzas@amd.com> Co-authored-by: OpenAI Codex <codex@openai.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
co-authored by
OpenAI Codex
mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent
e9e1630e93
commit
3a386cfaf5
+17
-1
@@ -1276,6 +1276,7 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
engine_core: EngineCoreProc | None = None
|
||||
signal_callback: SignalCallback | None = None
|
||||
clean_shutdown = False
|
||||
try:
|
||||
vllm_config: VllmConfig = kwargs["vllm_config"]
|
||||
parallel_config: ParallelConfig = vllm_config.parallel_config
|
||||
@@ -1338,8 +1339,15 @@ class EngineCoreProc(EngineCore):
|
||||
|
||||
engine_core.run_busy_loop()
|
||||
|
||||
except SystemExit:
|
||||
except SystemExit as e:
|
||||
logger.info_once("[shutdown] EngineCore: exiting busy loop")
|
||||
clean_shutdown = (
|
||||
e.code in (None, 0)
|
||||
and engine_core is not None
|
||||
and engine_core.shutdown_state == EngineShutdownState.SHUTTING_DOWN
|
||||
and not engine_core.has_work()
|
||||
and engine_core.vllm_config.shutdown_timeout == 0
|
||||
)
|
||||
raise
|
||||
except Exception as e:
|
||||
if engine_core is None:
|
||||
@@ -1355,6 +1363,14 @@ class EngineCoreProc(EngineCore):
|
||||
signal_callback.stop()
|
||||
if engine_core is not None:
|
||||
engine_core.shutdown()
|
||||
if clean_shutdown:
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
if current_platform.is_rocm():
|
||||
# Cleanup above already unfreezes and collects the heap.
|
||||
# Freeze the surviving graph to skip another slow cyclic-GC
|
||||
# scan during finalization; process exit reclaims it.
|
||||
gc.freeze()
|
||||
|
||||
def _init_data_parallel(self, vllm_config: VllmConfig):
|
||||
pass
|
||||
|
||||
+35
-1
@@ -40,6 +40,30 @@ if TYPE_CHECKING:
|
||||
logger = init_logger(__name__)
|
||||
|
||||
STARTUP_POLL_PERIOD_MS = 10000
|
||||
ROCM_ENGINE_PROCESS_SHUTDOWN_TIMEOUT_S = 15.0
|
||||
|
||||
|
||||
def get_engine_process_shutdown_timeout(
|
||||
request_timeout: float | None,
|
||||
process_timeout: float | None,
|
||||
) -> float | None:
|
||||
"""Return the EngineCore process-manager shutdown timeout.
|
||||
|
||||
``VllmConfig.shutdown_timeout`` controls how long in-flight requests may
|
||||
drain. A value of zero therefore tells EngineCore to abort requests as soon
|
||||
as it receives SIGTERM. The parent process manager still needs a separate
|
||||
window in which the EngineCore can release device resources before it is
|
||||
force-killed. ROCm teardown can take longer than the generic best-effort
|
||||
window, and force-killing during teardown can leave VRAM resident.
|
||||
|
||||
``process_timeout`` may be a remaining budget computed by an outer process
|
||||
manager. Keep it unchanged unless both values are zero: a zero remaining
|
||||
budget for a positive request timeout must not receive a fresh grace period
|
||||
because EngineCore relies on that deadline to enforce request draining.
|
||||
"""
|
||||
if request_timeout == 0 and process_timeout == 0 and current_platform.is_rocm():
|
||||
return ROCM_ENGINE_PROCESS_SHUTDOWN_TIMEOUT_S
|
||||
return process_timeout
|
||||
|
||||
|
||||
class CoreEngineState(Enum):
|
||||
@@ -136,6 +160,7 @@ class CoreEngineProcManager:
|
||||
client_handshake_address: str | None = None,
|
||||
tensor_queue: Queue | None = None,
|
||||
):
|
||||
self._request_shutdown_timeout = vllm_config.shutdown_timeout
|
||||
context = get_mp_context()
|
||||
common_kwargs = {
|
||||
"vllm_config": vllm_config,
|
||||
@@ -217,7 +242,16 @@ class CoreEngineProcManager:
|
||||
"""Shutdown engine core processes with configurable timeout."""
|
||||
self.manager_stopped.set()
|
||||
if self._finalizer.detach() is not None:
|
||||
shutdown(self.processes, timeout=timeout)
|
||||
process_timeout = get_engine_process_shutdown_timeout(
|
||||
self._request_shutdown_timeout, timeout
|
||||
)
|
||||
if process_timeout != timeout:
|
||||
logger.info(
|
||||
"[shutdown] EngineCore process manager: using %ss ROCm "
|
||||
"cleanup grace after immediate request abort",
|
||||
process_timeout,
|
||||
)
|
||||
shutdown(self.processes, timeout=process_timeout)
|
||||
|
||||
def monitor_engine_liveness(self) -> None:
|
||||
"""Monitor engine core process liveness."""
|
||||
|
||||
Reference in New Issue
Block a user