[PLE] The dying-path traceback runs no exception code

ple_fail_closed formatted its traceback with traceback.format_exception, which
runs exception-defined __str__/__getattribute__; if one blocked, the rank never
reached os._exit(1) and kept its GPUs. A safe formatter reads frames and plain
string arguments only.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:03:19 +07:00
co-authored by Hinata
parent 3575cdb126
commit a4aecf9ec7
+121 -4
View File
@@ -5,8 +5,8 @@
import functools import functools
import os import os
import time import time
import traceback
from collections.abc import Callable from collections.abc import Callable
from types import TracebackType
from typing import Any, NoReturn, TypeVar from typing import Any, NoReturn, TypeVar
_F = TypeVar("_F", bound=Callable[..., Any]) _F = TypeVar("_F", bound=Callable[..., Any])
@@ -26,6 +26,122 @@ _TRUSTED_ERROR_LABELS = (
(KeyboardInterrupt, "KeyboardInterrupt"), (KeyboardInterrupt, "KeyboardInterrupt"),
) )
# Slot readers for the dying path. Each is the C getter of the base type,
# called directly, so no __getattribute__, property, metaclass attribute or
# descriptor defined by the exception (or its class) can run. The traceback
# module is NOT safe here: traceback.format_exception calls str(error) (and
# str() on every chained exception) and reads error.__traceback__ through the
# instance's own __getattribute__, so an exception whose __str__ or
# __getattribute__ blocks kept the process alive forever -- the fail-closed
# path hung instead of dying (found by the blocking-exception transport tests).
_EXC_ARGS = BaseException.args.__get__
_EXC_TRACEBACK = BaseException.__traceback__.__get__
_EXC_CAUSE = BaseException.__cause__.__get__
_EXC_CONTEXT = BaseException.__context__.__get__
_EXC_SUPPRESS_CONTEXT = BaseException.__suppress_context__.__get__
_TYPE_QUALNAME = type.__dict__["__qualname__"].__get__
_TYPE_MODULE = type.__dict__["__module__"].__get__
_MAX_CHAIN = 16
_MAX_ARG_CHARS = 1024
def _safe_type_label(cls: type) -> str:
"""``module.qualname`` of ``cls`` without running class-defined code."""
try:
qualname = _TYPE_QUALNAME(cls)
module = _TYPE_MODULE(cls)
except BaseException:
return "<unknown type>"
if type(qualname) is not str:
qualname = "<unknown type>"
if type(module) is str and module != "builtins":
return f"{module}.{qualname}"
return qualname
def _safe_exception_line(error: BaseException) -> str:
"""Type label plus the exact-``str``/``int`` args; nothing else is shown."""
label = _safe_type_label(type(error))
args = _EXC_ARGS(error)
if type(args) is not tuple or len(args) == 0:
return label
parts = []
for arg in args[:4]:
if type(arg) is str:
parts.append(arg[:_MAX_ARG_CHARS])
elif type(arg) is int and -(10**18) < arg < 10**18:
parts.append(str(arg))
else:
parts.append(f"<{_safe_type_label(type(arg))}>")
if len(args) > 4:
parts.append("...")
if len(args) == 1:
return f"{label}: {parts[0]}"
return f"{label}: ({', '.join(parts)})"
def _safe_frames(error: BaseException) -> list[str]:
lines = []
tb = _EXC_TRACEBACK(error)
while type(tb) is TracebackType:
code = tb.tb_frame.f_code
filename = code.co_filename
name = code.co_qualname
lineno = tb.tb_lineno
lines.append(
f' File "{filename if type(filename) is str else "?"}", '
f"line {lineno if type(lineno) is int else '?'}, "
f"in {name if type(name) is str else '?'}\n"
)
tb = tb.tb_next
return lines
def _safe_format_exception(error: BaseException) -> str:
"""A traceback.format_exception look-alike that runs no exception code.
Frames come from the traceback objects (file, line, function; no source
lines, which would mean file I/O on the dying path); the message is the
exception's exact-``str`` args, never ``str(error)``. Chains follow
``__cause__`` / ``__context__`` like the interpreter, bounded and
cycle-safe.
"""
chain: list[tuple[BaseException, str]] = []
seen: set[int] = set()
current: BaseException | None = error
link = ""
while current is not None and len(chain) < _MAX_CHAIN:
if id(current) in seen:
break
seen.add(id(current))
chain.append((current, link))
cause = _EXC_CAUSE(current)
if cause is not None:
current = cause
link = (
"\nThe above exception was the direct cause of the "
"following exception:\n\n"
)
elif _EXC_SUPPRESS_CONTEXT(current) is not True:
current = _EXC_CONTEXT(current)
link = (
"\nDuring handling of the above exception, another "
"exception occurred:\n\n"
)
else:
current = None
out: list[str] = []
# Innermost first, as the interpreter prints; each outer exception's link
# text sits between it and the exception it chains to.
for index in range(len(chain) - 1, -1, -1):
exc, _ = chain[index]
out.append("Traceback (most recent call last):\n")
out.extend(_safe_frames(exc))
out.append(_safe_exception_line(exc) + "\n")
if index > 0:
out.append(chain[index][1])
return "".join(out)
def ple_fail_closed( def ple_fail_closed(
*, *,
@@ -88,9 +204,10 @@ def ple_fail_closed(
# Three-arg form: it does not depend on being inside an active except # Three-arg form: it does not depend on being inside an active except
# block, and this function is called from one frame further out than # block, and this function is called from one frame further out than
# the raise. # the raise.
tb = "".join( # _safe_format_exception, not traceback.format_exception: see the note
traceback.format_exception(type(error), error, error.__traceback__) # above _EXC_ARGS -- the stdlib formatter calls str(error), which is
) # exception-defined code and can block the dying process forever.
tb = _safe_format_exception(error)
raw = tb.encode("utf-8", "backslashreplace") raw = tb.encode("utf-8", "backslashreplace")
# Truncate from the END, not the start. format_exception puts the # Truncate from the END, not the start. format_exception puts the
# frames first and the exception TYPE AND MESSAGE last, so a head-side # frames first and the exception TYPE AND MESSAGE last, so a head-side