[Bugfix] Log CRIU failure details before snapshot cleanup (#59661)

Signed-off-by: Nils Matteson <nilsmatteson@icloud.com>
This commit is contained in:
Nils Matteson
2026-10-03 04:02:27 +00:00
committed by GitHub
parent 9fdb174678
commit 28556f8600
2 changed files with 152 additions and 1 deletions
@@ -719,6 +719,111 @@ def test_snapshot_create_rolls_back_failed_dump(tmp_path: Path):
assert not target.exists()
@pytest.mark.parametrize(
"failure_kind", ["called-process", "timeout", "missing", "unreadable"]
)
def test_snapshot_create_preserves_bounded_criu_diagnostics_after_cleanup(
tmp_path: Path, failure_kind: str, caplog: pytest.LogCaptureFixture
):
target = tmp_path / "snapshot"
tools = _fake_snapshot_tools()
criu_tools = LocalSnapshotTools()
criu_tools.plugin_dir = tmp_path
original_run = criu_tools._run
criu_tools._privileged = lambda: [] # type: ignore[method-assign]
captured = "captured stderr " + "x" * 9000
if failure_kind == "timeout":
primary: subprocess.SubprocessError = subprocess.TimeoutExpired(
["criu", "dump"], 1, output=b"captured stdout\xff", stderr=captured.encode()
)
else:
primary = subprocess.CalledProcessError(
1, ["criu", "dump"], output="captured stdout", stderr=captured
)
def run(command: list[str], **_kwargs: object):
if "tail" not in command:
raise primary
if failure_kind == "unreadable":
raise PermissionError("log is root-owned")
if failure_kind == "timeout":
return subprocess.CompletedProcess(command, 0, "timeout log tail", "")
return original_run(command, **_kwargs)
criu_tools._run = run # type: ignore[method-assign]
def fail_dump(workdir: Path, _inventory: ProcessInventory) -> None:
images = workdir / "images"
images.mkdir()
if failure_kind != "missing":
(images / "dump.log").write_text(
"omitted dump prefix" + "d" * 9000 + "distinctive dump diagnostic"
)
criu_tools._criu("dump", workdir, [])
tools.dump.side_effect = fail_dump
with pytest.raises(type(primary)) as excinfo:
create_snapshot(
argparse.Namespace(snapshot_dir=str(target), model_tag="Qwen/Qwen3-0.6B"),
tools=tools,
)
assert excinfo.value is primary
assert not target.exists()
diagnostics = "\n".join(caplog.messages)
assert "captured stdout" in diagnostics
assert "captured stderr" not in diagnostics
assert "x" * 100 in diagnostics
assert len(diagnostics) < 17000
if failure_kind in {"missing", "unreadable"}:
assert "dump.log unavailable" in diagnostics
expected_error = (
"CalledProcessError" if failure_kind == "missing" else "PermissionError"
)
assert expected_error in diagnostics
elif failure_kind == "called-process":
assert "distinctive dump diagnostic" in diagnostics
assert "omitted dump prefix" not in diagnostics
else:
assert "timeout log tail" in diagnostics
if failure_kind == "timeout":
assert "captured stdout�" in diagnostics
def test_snapshot_criu_failure_selects_restore_log(
tmp_path: Path, caplog: pytest.LogCaptureFixture
):
tools = LocalSnapshotTools()
tools.plugin_dir = tmp_path
tools._privileged = lambda: [] # type: ignore[method-assign]
artifact = tmp_path / "snapshot"
images = artifact / "images"
images.mkdir(parents=True)
(images / "restore.log").write_text(
"omitted restore prefix" + "r" * 9000 + "distinctive restore diagnostic"
)
primary = subprocess.CalledProcessError(1, ["criu", "restore"])
original_run = tools._run
tailed: list[str] = []
def run(command: list[str], **_kwargs: object):
if "tail" not in command:
raise primary
tailed.append(command[-1])
return original_run(command, **_kwargs)
tools._run = run # type: ignore[method-assign]
with pytest.raises(subprocess.CalledProcessError) as excinfo:
tools._criu("restore", artifact, [])
assert excinfo.value is primary
assert tailed == [str(images / "restore.log")]
diagnostics = "\n".join(caplog.messages)
assert "distinctive restore diagnostic" in diagnostics
assert "omitted restore prefix" not in diagnostics
def _restore_fixture(tmp_path: Path) -> argparse.Namespace:
artifact = tmp_path / "snapshot"
artifact.mkdir(mode=0o700)
+47 -1
View File
@@ -31,6 +31,7 @@ from typing import NoReturn
import regex as re
from pydantic import ValidationError
from vllm.logger import init_logger
from vllm.snapshot.manifest import (
ReadyMarker,
SnapshotManifest,
@@ -43,6 +44,8 @@ from vllm.snapshot.manifest import (
)
from vllm.snapshot.types import Oracle
logger = init_logger(__name__)
class SnapshotCreateError(RuntimeError):
"""Snapshot creation did not produce a complete artifact."""
@@ -59,6 +62,7 @@ _COMMON_CRIU_OPTIONS = (
"--link-remap",
"--file-locks",
)
_CRIU_DIAGNOSTIC_LIMIT = 8192
_TCP_TABLES = (
("AF_INET", Path("/proc/net/tcp")),
("AF_INET6", Path("/proc/net/tcp6")),
@@ -503,7 +507,49 @@ class LocalSnapshotTools:
str(self.plugin_dir),
*arguments,
]
self._run(command)
try:
self._run(command)
except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error:
self._log_criu_diagnostics(error, action, artifact)
raise
def _log_criu_diagnostics(
self,
error: subprocess.CalledProcessError | subprocess.TimeoutExpired,
action: str,
artifact: Path,
) -> None:
def bounded(value: str | bytes) -> str:
if isinstance(value, bytes):
return value[-_CRIU_DIAGNOSTIC_LIMIT:].decode(errors="replace")
return value[-_CRIU_DIAGNOSTIC_LIMIT:]
for name in ("stdout", "stderr"):
value = getattr(error, name, None)
if value:
logger.error("CRIU %s (tail):\n%s", name, bounded(value))
log_path = artifact / "images" / f"{action}.log"
try:
result = self._run(
[
*self._privileged(),
"tail",
"-c",
str(_CRIU_DIAGNOSTIC_LIMIT),
"--",
str(log_path),
],
timeout=5,
)
if result.stdout:
logger.error("CRIU %s.log (tail):\n%s", action, bounded(result.stdout))
except BaseException as diagnostic_error:
logger.error(
"CRIU %s.log unavailable: %s",
action,
bounded(_error_detail(diagnostic_error)),
)
def _record_child_log_size(self, artifact: Path) -> None:
child_log = artifact / "child.log"