[Bugfix] Keep force-killing the process tree when a child exits mid-walk (#59984)
Signed-off-by: Lanze Liu <lanzetech@gmail.com> Co-authored-by: Ning Xie <andy.xning@gmail.com>
This commit is contained in:
@@ -4,10 +4,15 @@
|
||||
import contextlib
|
||||
import io
|
||||
import os
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import psutil
|
||||
|
||||
from vllm.utils import system_utils
|
||||
from vllm.utils.system_utils import (
|
||||
_maybe_force_spawn,
|
||||
suppress_stdout,
|
||||
@@ -48,3 +53,28 @@ def test_suppress_stdout_keeps_stderr_when_sys_stdout_is_stderr(capfd, monkeypat
|
||||
out, err = capfd.readouterr()
|
||||
assert "c library output" not in out
|
||||
assert "error message" in err
|
||||
|
||||
|
||||
def test_kill_process_tree_skips_child_that_exits_mid_walk(monkeypatch):
|
||||
sleep = [sys.executable, "-c", "import time; time.sleep(60)"]
|
||||
target, *children = procs = [subprocess.Popen(sleep) for _ in range(4)]
|
||||
try:
|
||||
# List the children while they are alive, then let one exit and be
|
||||
# reaped before the walk reaches it, so psutil's name() raises.
|
||||
listed = [psutil.Process(p.pid) for p in children]
|
||||
children[1].terminate()
|
||||
children[1].wait(timeout=10)
|
||||
monkeypatch.setattr(
|
||||
psutil.Process, "children", lambda self, recursive=False: listed
|
||||
)
|
||||
|
||||
system_utils.kill_process_tree(target.pid)
|
||||
|
||||
# The exited child died from our SIGTERM; the rest were SIGKILLed.
|
||||
assert children[1].returncode == -signal.SIGTERM
|
||||
for p in (children[0], children[2], target):
|
||||
assert p.wait(timeout=10) == -signal.SIGKILL
|
||||
finally:
|
||||
for p in procs:
|
||||
p.kill()
|
||||
p.wait()
|
||||
|
||||
@@ -221,7 +221,7 @@ def kill_process_tree(pid: int):
|
||||
|
||||
# Send SIGKILL to all children first
|
||||
for child in children:
|
||||
with contextlib.suppress(ProcessLookupError):
|
||||
with contextlib.suppress(ProcessLookupError, psutil.NoSuchProcess):
|
||||
logger.warning(
|
||||
"[shutdown] force killing sub process %s pid %d",
|
||||
child.name(),
|
||||
|
||||
Reference in New Issue
Block a user