diff --git a/tests/entrypoints/launchers/test_launch_cli.py b/tests/entrypoints/launchers/test_launch_cli.py index 1eb27d5a62..55093c5685 100644 --- a/tests/entrypoints/launchers/test_launch_cli.py +++ b/tests/entrypoints/launchers/test_launch_cli.py @@ -719,6 +719,111 @@ def test_snapshot_create_rolls_back_failed_dump(tmp_path: Path): assert not target.exists() +@pytest.mark.parametrize( + "failure_kind", ["called-process", "timeout", "missing", "unreadable"] +) +def test_snapshot_create_preserves_bounded_criu_diagnostics_after_cleanup( + tmp_path: Path, failure_kind: str, caplog: pytest.LogCaptureFixture +): + target = tmp_path / "snapshot" + tools = _fake_snapshot_tools() + criu_tools = LocalSnapshotTools() + criu_tools.plugin_dir = tmp_path + original_run = criu_tools._run + criu_tools._privileged = lambda: [] # type: ignore[method-assign] + captured = "captured stderr " + "x" * 9000 + if failure_kind == "timeout": + primary: subprocess.SubprocessError = subprocess.TimeoutExpired( + ["criu", "dump"], 1, output=b"captured stdout\xff", stderr=captured.encode() + ) + else: + primary = subprocess.CalledProcessError( + 1, ["criu", "dump"], output="captured stdout", stderr=captured + ) + + def run(command: list[str], **_kwargs: object): + if "tail" not in command: + raise primary + if failure_kind == "unreadable": + raise PermissionError("log is root-owned") + if failure_kind == "timeout": + return subprocess.CompletedProcess(command, 0, "timeout log tail", "") + return original_run(command, **_kwargs) + + criu_tools._run = run # type: ignore[method-assign] + + def fail_dump(workdir: Path, _inventory: ProcessInventory) -> None: + images = workdir / "images" + images.mkdir() + if failure_kind != "missing": + (images / "dump.log").write_text( + "omitted dump prefix" + "d" * 9000 + "distinctive dump diagnostic" + ) + criu_tools._criu("dump", workdir, []) + + tools.dump.side_effect = fail_dump + with pytest.raises(type(primary)) as excinfo: + create_snapshot( + argparse.Namespace(snapshot_dir=str(target), model_tag="Qwen/Qwen3-0.6B"), + tools=tools, + ) + + assert excinfo.value is primary + assert not target.exists() + diagnostics = "\n".join(caplog.messages) + assert "captured stdout" in diagnostics + assert "captured stderr" not in diagnostics + assert "x" * 100 in diagnostics + assert len(diagnostics) < 17000 + if failure_kind in {"missing", "unreadable"}: + assert "dump.log unavailable" in diagnostics + expected_error = ( + "CalledProcessError" if failure_kind == "missing" else "PermissionError" + ) + assert expected_error in diagnostics + elif failure_kind == "called-process": + assert "distinctive dump diagnostic" in diagnostics + assert "omitted dump prefix" not in diagnostics + else: + assert "timeout log tail" in diagnostics + if failure_kind == "timeout": + assert "captured stdout�" in diagnostics + + +def test_snapshot_criu_failure_selects_restore_log( + tmp_path: Path, caplog: pytest.LogCaptureFixture +): + tools = LocalSnapshotTools() + tools.plugin_dir = tmp_path + tools._privileged = lambda: [] # type: ignore[method-assign] + artifact = tmp_path / "snapshot" + images = artifact / "images" + images.mkdir(parents=True) + (images / "restore.log").write_text( + "omitted restore prefix" + "r" * 9000 + "distinctive restore diagnostic" + ) + primary = subprocess.CalledProcessError(1, ["criu", "restore"]) + original_run = tools._run + tailed: list[str] = [] + + def run(command: list[str], **_kwargs: object): + if "tail" not in command: + raise primary + tailed.append(command[-1]) + return original_run(command, **_kwargs) + + tools._run = run # type: ignore[method-assign] + + with pytest.raises(subprocess.CalledProcessError) as excinfo: + tools._criu("restore", artifact, []) + + assert excinfo.value is primary + assert tailed == [str(images / "restore.log")] + diagnostics = "\n".join(caplog.messages) + assert "distinctive restore diagnostic" in diagnostics + assert "omitted restore prefix" not in diagnostics + + def _restore_fixture(tmp_path: Path) -> argparse.Namespace: artifact = tmp_path / "snapshot" artifact.mkdir(mode=0o700) diff --git a/vllm/snapshot/runtime.py b/vllm/snapshot/runtime.py index 8fb1c7b526..fdced5116c 100644 --- a/vllm/snapshot/runtime.py +++ b/vllm/snapshot/runtime.py @@ -31,6 +31,7 @@ from typing import NoReturn import regex as re from pydantic import ValidationError +from vllm.logger import init_logger from vllm.snapshot.manifest import ( ReadyMarker, SnapshotManifest, @@ -43,6 +44,8 @@ from vllm.snapshot.manifest import ( ) from vllm.snapshot.types import Oracle +logger = init_logger(__name__) + class SnapshotCreateError(RuntimeError): """Snapshot creation did not produce a complete artifact.""" @@ -59,6 +62,7 @@ _COMMON_CRIU_OPTIONS = ( "--link-remap", "--file-locks", ) +_CRIU_DIAGNOSTIC_LIMIT = 8192 _TCP_TABLES = ( ("AF_INET", Path("/proc/net/tcp")), ("AF_INET6", Path("/proc/net/tcp6")), @@ -503,7 +507,49 @@ class LocalSnapshotTools: str(self.plugin_dir), *arguments, ] - self._run(command) + try: + self._run(command) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as error: + self._log_criu_diagnostics(error, action, artifact) + raise + + def _log_criu_diagnostics( + self, + error: subprocess.CalledProcessError | subprocess.TimeoutExpired, + action: str, + artifact: Path, + ) -> None: + def bounded(value: str | bytes) -> str: + if isinstance(value, bytes): + return value[-_CRIU_DIAGNOSTIC_LIMIT:].decode(errors="replace") + return value[-_CRIU_DIAGNOSTIC_LIMIT:] + + for name in ("stdout", "stderr"): + value = getattr(error, name, None) + if value: + logger.error("CRIU %s (tail):\n%s", name, bounded(value)) + + log_path = artifact / "images" / f"{action}.log" + try: + result = self._run( + [ + *self._privileged(), + "tail", + "-c", + str(_CRIU_DIAGNOSTIC_LIMIT), + "--", + str(log_path), + ], + timeout=5, + ) + if result.stdout: + logger.error("CRIU %s.log (tail):\n%s", action, bounded(result.stdout)) + except BaseException as diagnostic_error: + logger.error( + "CRIU %s.log unavailable: %s", + action, + bounded(_error_detail(diagnostic_error)), + ) def _record_child_log_size(self, artifact: Path) -> None: child_log = artifact / "child.log"