[Bugfix] Log CRIU failure details before snapshot cleanup (#59661)
Signed-off-by: Nils Matteson <nilsmatteson@icloud.com>
This commit is contained in:
@@ -719,6 +719,111 @@ def test_snapshot_create_rolls_back_failed_dump(tmp_path: Path):
|
||||
assert not target.exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"failure_kind", ["called-process", "timeout", "missing", "unreadable"]
|
||||
)
|
||||
def test_snapshot_create_preserves_bounded_criu_diagnostics_after_cleanup(
|
||||
tmp_path: Path, failure_kind: str, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
target = tmp_path / "snapshot"
|
||||
tools = _fake_snapshot_tools()
|
||||
criu_tools = LocalSnapshotTools()
|
||||
criu_tools.plugin_dir = tmp_path
|
||||
original_run = criu_tools._run
|
||||
criu_tools._privileged = lambda: [] # type: ignore[method-assign]
|
||||
captured = "captured stderr " + "x" * 9000
|
||||
if failure_kind == "timeout":
|
||||
primary: subprocess.SubprocessError = subprocess.TimeoutExpired(
|
||||
["criu", "dump"], 1, output=b"captured stdout\xff", stderr=captured.encode()
|
||||
)
|
||||
else:
|
||||
primary = subprocess.CalledProcessError(
|
||||
1, ["criu", "dump"], output="captured stdout", stderr=captured
|
||||
)
|
||||
|
||||
def run(command: list[str], **_kwargs: object):
|
||||
if "tail" not in command:
|
||||
raise primary
|
||||
if failure_kind == "unreadable":
|
||||
raise PermissionError("log is root-owned")
|
||||
if failure_kind == "timeout":
|
||||
return subprocess.CompletedProcess(command, 0, "timeout log tail", "")
|
||||
return original_run(command, **_kwargs)
|
||||
|
||||
criu_tools._run = run # type: ignore[method-assign]
|
||||
|
||||
def fail_dump(workdir: Path, _inventory: ProcessInventory) -> None:
|
||||
images = workdir / "images"
|
||||
images.mkdir()
|
||||
if failure_kind != "missing":
|
||||
(images / "dump.log").write_text(
|
||||
"omitted dump prefix" + "d" * 9000 + "distinctive dump diagnostic"
|
||||
)
|
||||
criu_tools._criu("dump", workdir, [])
|
||||
|
||||
tools.dump.side_effect = fail_dump
|
||||
with pytest.raises(type(primary)) as excinfo:
|
||||
create_snapshot(
|
||||
argparse.Namespace(snapshot_dir=str(target), model_tag="Qwen/Qwen3-0.6B"),
|
||||
tools=tools,
|
||||
)
|
||||
|
||||
assert excinfo.value is primary
|
||||
assert not target.exists()
|
||||
diagnostics = "\n".join(caplog.messages)
|
||||
assert "captured stdout" in diagnostics
|
||||
assert "captured stderr" not in diagnostics
|
||||
assert "x" * 100 in diagnostics
|
||||
assert len(diagnostics) < 17000
|
||||
if failure_kind in {"missing", "unreadable"}:
|
||||
assert "dump.log unavailable" in diagnostics
|
||||
expected_error = (
|
||||
"CalledProcessError" if failure_kind == "missing" else "PermissionError"
|
||||
)
|
||||
assert expected_error in diagnostics
|
||||
elif failure_kind == "called-process":
|
||||
assert "distinctive dump diagnostic" in diagnostics
|
||||
assert "omitted dump prefix" not in diagnostics
|
||||
else:
|
||||
assert "timeout log tail" in diagnostics
|
||||
if failure_kind == "timeout":
|
||||
assert "captured stdout�" in diagnostics
|
||||
|
||||
|
||||
def test_snapshot_criu_failure_selects_restore_log(
|
||||
tmp_path: Path, caplog: pytest.LogCaptureFixture
|
||||
):
|
||||
tools = LocalSnapshotTools()
|
||||
tools.plugin_dir = tmp_path
|
||||
tools._privileged = lambda: [] # type: ignore[method-assign]
|
||||
artifact = tmp_path / "snapshot"
|
||||
images = artifact / "images"
|
||||
images.mkdir(parents=True)
|
||||
(images / "restore.log").write_text(
|
||||
"omitted restore prefix" + "r" * 9000 + "distinctive restore diagnostic"
|
||||
)
|
||||
primary = subprocess.CalledProcessError(1, ["criu", "restore"])
|
||||
original_run = tools._run
|
||||
tailed: list[str] = []
|
||||
|
||||
def run(command: list[str], **_kwargs: object):
|
||||
if "tail" not in command:
|
||||
raise primary
|
||||
tailed.append(command[-1])
|
||||
return original_run(command, **_kwargs)
|
||||
|
||||
tools._run = run # type: ignore[method-assign]
|
||||
|
||||
with pytest.raises(subprocess.CalledProcessError) as excinfo:
|
||||
tools._criu("restore", artifact, [])
|
||||
|
||||
assert excinfo.value is primary
|
||||
assert tailed == [str(images / "restore.log")]
|
||||
diagnostics = "\n".join(caplog.messages)
|
||||
assert "distinctive restore diagnostic" in diagnostics
|
||||
assert "omitted restore prefix" not in diagnostics
|
||||
|
||||
|
||||
def _restore_fixture(tmp_path: Path) -> argparse.Namespace:
|
||||
artifact = tmp_path / "snapshot"
|
||||
artifact.mkdir(mode=0o700)
|
||||
|
||||
@@ -31,6 +31,7 @@ from typing import NoReturn
|
||||
import regex as re
|
||||
from pydantic import ValidationError
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.snapshot.manifest import (
|
||||
ReadyMarker,
|
||||
SnapshotManifest,
|
||||
@@ -43,6 +44,8 @@ from vllm.snapshot.manifest import (
|
||||
)
|
||||
from vllm.snapshot.types import Oracle
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class SnapshotCreateError(RuntimeError):
|
||||
"""Snapshot creation did not produce a complete artifact."""
|
||||
@@ -59,6 +62,7 @@ _COMMON_CRIU_OPTIONS = (
|
||||
"--link-remap",
|
||||
"--file-locks",
|
||||
)
|
||||
_CRIU_DIAGNOSTIC_LIMIT = 8192
|
||||
_TCP_TABLES = (
|
||||
("AF_INET", Path("/proc/net/tcp")),
|
||||
("AF_INET6", Path("/proc/net/tcp6")),
|
||||
@@ -503,7 +507,49 @@ class LocalSnapshotTools:
|
||||
str(self.plugin_dir),
|
||||
*arguments,
|
||||
]
|
||||
self._run(command)
|
||||
try:
|
||||
self._run(command)
|
||||
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
|
||||
self._log_criu_diagnostics(error, action, artifact)
|
||||
raise
|
||||
|
||||
def _log_criu_diagnostics(
|
||||
self,
|
||||
error: subprocess.CalledProcessError | subprocess.TimeoutExpired,
|
||||
action: str,
|
||||
artifact: Path,
|
||||
) -> None:
|
||||
def bounded(value: str | bytes) -> str:
|
||||
if isinstance(value, bytes):
|
||||
return value[-_CRIU_DIAGNOSTIC_LIMIT:].decode(errors="replace")
|
||||
return value[-_CRIU_DIAGNOSTIC_LIMIT:]
|
||||
|
||||
for name in ("stdout", "stderr"):
|
||||
value = getattr(error, name, None)
|
||||
if value:
|
||||
logger.error("CRIU %s (tail):\n%s", name, bounded(value))
|
||||
|
||||
log_path = artifact / "images" / f"{action}.log"
|
||||
try:
|
||||
result = self._run(
|
||||
[
|
||||
*self._privileged(),
|
||||
"tail",
|
||||
"-c",
|
||||
str(_CRIU_DIAGNOSTIC_LIMIT),
|
||||
"--",
|
||||
str(log_path),
|
||||
],
|
||||
timeout=5,
|
||||
)
|
||||
if result.stdout:
|
||||
logger.error("CRIU %s.log (tail):\n%s", action, bounded(result.stdout))
|
||||
except BaseException as diagnostic_error:
|
||||
logger.error(
|
||||
"CRIU %s.log unavailable: %s",
|
||||
action,
|
||||
bounded(_error_detail(diagnostic_error)),
|
||||
)
|
||||
|
||||
def _record_child_log_size(self, artifact: Path) -> None:
|
||||
child_log = artifact / "child.log"
|
||||
|
||||
Reference in New Issue
Block a user