[PLE] The dying-path traceback runs no exception code
ple_fail_closed formatted its traceback with traceback.format_exception, which runs exception-defined __str__/__getattribute__; if one blocked, the rank never reached os._exit(1) and kept its GPUs. A safe formatter reads frames and plain string arguments only. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -5,8 +5,8 @@
|
|||||||
import functools
|
import functools
|
||||||
import os
|
import os
|
||||||
import time
|
import time
|
||||||
import traceback
|
|
||||||
from collections.abc import Callable
|
from collections.abc import Callable
|
||||||
|
from types import TracebackType
|
||||||
from typing import Any, NoReturn, TypeVar
|
from typing import Any, NoReturn, TypeVar
|
||||||
|
|
||||||
_F = TypeVar("_F", bound=Callable[..., Any])
|
_F = TypeVar("_F", bound=Callable[..., Any])
|
||||||
@@ -26,6 +26,122 @@ _TRUSTED_ERROR_LABELS = (
|
|||||||
(KeyboardInterrupt, "KeyboardInterrupt"),
|
(KeyboardInterrupt, "KeyboardInterrupt"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
# Slot readers for the dying path. Each is the C getter of the base type,
|
||||||
|
# called directly, so no __getattribute__, property, metaclass attribute or
|
||||||
|
# descriptor defined by the exception (or its class) can run. The traceback
|
||||||
|
# module is NOT safe here: traceback.format_exception calls str(error) (and
|
||||||
|
# str() on every chained exception) and reads error.__traceback__ through the
|
||||||
|
# instance's own __getattribute__, so an exception whose __str__ or
|
||||||
|
# __getattribute__ blocks kept the process alive forever -- the fail-closed
|
||||||
|
# path hung instead of dying (found by the blocking-exception transport tests).
|
||||||
|
_EXC_ARGS = BaseException.args.__get__
|
||||||
|
_EXC_TRACEBACK = BaseException.__traceback__.__get__
|
||||||
|
_EXC_CAUSE = BaseException.__cause__.__get__
|
||||||
|
_EXC_CONTEXT = BaseException.__context__.__get__
|
||||||
|
_EXC_SUPPRESS_CONTEXT = BaseException.__suppress_context__.__get__
|
||||||
|
_TYPE_QUALNAME = type.__dict__["__qualname__"].__get__
|
||||||
|
_TYPE_MODULE = type.__dict__["__module__"].__get__
|
||||||
|
_MAX_CHAIN = 16
|
||||||
|
_MAX_ARG_CHARS = 1024
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_type_label(cls: type) -> str:
|
||||||
|
"""``module.qualname`` of ``cls`` without running class-defined code."""
|
||||||
|
try:
|
||||||
|
qualname = _TYPE_QUALNAME(cls)
|
||||||
|
module = _TYPE_MODULE(cls)
|
||||||
|
except BaseException:
|
||||||
|
return "<unknown type>"
|
||||||
|
if type(qualname) is not str:
|
||||||
|
qualname = "<unknown type>"
|
||||||
|
if type(module) is str and module != "builtins":
|
||||||
|
return f"{module}.{qualname}"
|
||||||
|
return qualname
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_exception_line(error: BaseException) -> str:
|
||||||
|
"""Type label plus the exact-``str``/``int`` args; nothing else is shown."""
|
||||||
|
label = _safe_type_label(type(error))
|
||||||
|
args = _EXC_ARGS(error)
|
||||||
|
if type(args) is not tuple or len(args) == 0:
|
||||||
|
return label
|
||||||
|
parts = []
|
||||||
|
for arg in args[:4]:
|
||||||
|
if type(arg) is str:
|
||||||
|
parts.append(arg[:_MAX_ARG_CHARS])
|
||||||
|
elif type(arg) is int and -(10**18) < arg < 10**18:
|
||||||
|
parts.append(str(arg))
|
||||||
|
else:
|
||||||
|
parts.append(f"<{_safe_type_label(type(arg))}>")
|
||||||
|
if len(args) > 4:
|
||||||
|
parts.append("...")
|
||||||
|
if len(args) == 1:
|
||||||
|
return f"{label}: {parts[0]}"
|
||||||
|
return f"{label}: ({', '.join(parts)})"
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_frames(error: BaseException) -> list[str]:
|
||||||
|
lines = []
|
||||||
|
tb = _EXC_TRACEBACK(error)
|
||||||
|
while type(tb) is TracebackType:
|
||||||
|
code = tb.tb_frame.f_code
|
||||||
|
filename = code.co_filename
|
||||||
|
name = code.co_qualname
|
||||||
|
lineno = tb.tb_lineno
|
||||||
|
lines.append(
|
||||||
|
f' File "{filename if type(filename) is str else "?"}", '
|
||||||
|
f"line {lineno if type(lineno) is int else '?'}, "
|
||||||
|
f"in {name if type(name) is str else '?'}\n"
|
||||||
|
)
|
||||||
|
tb = tb.tb_next
|
||||||
|
return lines
|
||||||
|
|
||||||
|
|
||||||
|
def _safe_format_exception(error: BaseException) -> str:
|
||||||
|
"""A traceback.format_exception look-alike that runs no exception code.
|
||||||
|
|
||||||
|
Frames come from the traceback objects (file, line, function; no source
|
||||||
|
lines, which would mean file I/O on the dying path); the message is the
|
||||||
|
exception's exact-``str`` args, never ``str(error)``. Chains follow
|
||||||
|
``__cause__`` / ``__context__`` like the interpreter, bounded and
|
||||||
|
cycle-safe.
|
||||||
|
"""
|
||||||
|
chain: list[tuple[BaseException, str]] = []
|
||||||
|
seen: set[int] = set()
|
||||||
|
current: BaseException | None = error
|
||||||
|
link = ""
|
||||||
|
while current is not None and len(chain) < _MAX_CHAIN:
|
||||||
|
if id(current) in seen:
|
||||||
|
break
|
||||||
|
seen.add(id(current))
|
||||||
|
chain.append((current, link))
|
||||||
|
cause = _EXC_CAUSE(current)
|
||||||
|
if cause is not None:
|
||||||
|
current = cause
|
||||||
|
link = (
|
||||||
|
"\nThe above exception was the direct cause of the "
|
||||||
|
"following exception:\n\n"
|
||||||
|
)
|
||||||
|
elif _EXC_SUPPRESS_CONTEXT(current) is not True:
|
||||||
|
current = _EXC_CONTEXT(current)
|
||||||
|
link = (
|
||||||
|
"\nDuring handling of the above exception, another "
|
||||||
|
"exception occurred:\n\n"
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
current = None
|
||||||
|
out: list[str] = []
|
||||||
|
# Innermost first, as the interpreter prints; each outer exception's link
|
||||||
|
# text sits between it and the exception it chains to.
|
||||||
|
for index in range(len(chain) - 1, -1, -1):
|
||||||
|
exc, _ = chain[index]
|
||||||
|
out.append("Traceback (most recent call last):\n")
|
||||||
|
out.extend(_safe_frames(exc))
|
||||||
|
out.append(_safe_exception_line(exc) + "\n")
|
||||||
|
if index > 0:
|
||||||
|
out.append(chain[index][1])
|
||||||
|
return "".join(out)
|
||||||
|
|
||||||
|
|
||||||
def ple_fail_closed(
|
def ple_fail_closed(
|
||||||
*,
|
*,
|
||||||
@@ -88,9 +204,10 @@ def ple_fail_closed(
|
|||||||
# Three-arg form: it does not depend on being inside an active except
|
# Three-arg form: it does not depend on being inside an active except
|
||||||
# block, and this function is called from one frame further out than
|
# block, and this function is called from one frame further out than
|
||||||
# the raise.
|
# the raise.
|
||||||
tb = "".join(
|
# _safe_format_exception, not traceback.format_exception: see the note
|
||||||
traceback.format_exception(type(error), error, error.__traceback__)
|
# above _EXC_ARGS -- the stdlib formatter calls str(error), which is
|
||||||
)
|
# exception-defined code and can block the dying process forever.
|
||||||
|
tb = _safe_format_exception(error)
|
||||||
raw = tb.encode("utf-8", "backslashreplace")
|
raw = tb.encode("utf-8", "backslashreplace")
|
||||||
# Truncate from the END, not the start. format_exception puts the
|
# Truncate from the END, not the start. format_exception puts the
|
||||||
# frames first and the exception TYPE AND MESSAGE last, so a head-side
|
# frames first and the exception TYPE AND MESSAGE last, so a head-side
|
||||||
|
|||||||
Reference in New Issue
Block a user