[PLE] The dying-path traceback runs no exception code

ple_fail_closed formatted its traceback with traceback.format_exception, which
runs exception-defined __str__/__getattribute__; if one blocked, the rank never
reached os._exit(1) and kept its GPUs. A safe formatter reads frames and plain
string arguments only.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>

vllm/v1/ple_offload/fatal.py

@@ -5,8 +5,8 @@
 import functools
 import os
 import time
-import traceback
 from collections.abc import Callable
+from types import TracebackType
 from typing import Any, NoReturn, TypeVar
 
 _F = TypeVar("_F", bound=Callable[..., Any])
@@ -26,6 +26,122 @@ _TRUSTED_ERROR_LABELS = (
     (KeyboardInterrupt, "KeyboardInterrupt"),
 )
 
+# Slot readers for the dying path. Each is the C getter of the base type,
+# called directly, so no __getattribute__, property, metaclass attribute or
+# descriptor defined by the exception (or its class) can run. The traceback
+# module is NOT safe here: traceback.format_exception calls str(error) (and
+# str() on every chained exception) and reads error.__traceback__ through the
+# instance's own __getattribute__, so an exception whose __str__ or
+# __getattribute__ blocks kept the process alive forever -- the fail-closed
+# path hung instead of dying (found by the blocking-exception transport tests).
+_EXC_ARGS = BaseException.args.__get__
+_EXC_TRACEBACK = BaseException.__traceback__.__get__
+_EXC_CAUSE = BaseException.__cause__.__get__
+_EXC_CONTEXT = BaseException.__context__.__get__
+_EXC_SUPPRESS_CONTEXT = BaseException.__suppress_context__.__get__
+_TYPE_QUALNAME = type.__dict__["__qualname__"].__get__
+_TYPE_MODULE = type.__dict__["__module__"].__get__
+_MAX_CHAIN = 16
+_MAX_ARG_CHARS = 1024
+
+
+def _safe_type_label(cls: type) -> str:
+    """``module.qualname`` of ``cls`` without running class-defined code."""
+    try:
+        qualname = _TYPE_QUALNAME(cls)
+        module = _TYPE_MODULE(cls)
+    except BaseException:
+        return "<unknown type>"
+    if type(qualname) is not str:
+        qualname = "<unknown type>"
+    if type(module) is str and module != "builtins":
+        return f"{module}.{qualname}"
+    return qualname
+
+
+def _safe_exception_line(error: BaseException) -> str:
+    """Type label plus the exact-``str``/``int`` args; nothing else is shown."""
+    label = _safe_type_label(type(error))
+    args = _EXC_ARGS(error)
+    if type(args) is not tuple or len(args) == 0:
+        return label
+    parts = []
+    for arg in args[:4]:
+        if type(arg) is str:
+            parts.append(arg[:_MAX_ARG_CHARS])
+        elif type(arg) is int and -(10**18) < arg < 10**18:
+            parts.append(str(arg))
+        else:
+            parts.append(f"<{_safe_type_label(type(arg))}>")
+    if len(args) > 4:
+        parts.append("...")
+    if len(args) == 1:
+        return f"{label}: {parts[0]}"
+    return f"{label}: ({', '.join(parts)})"
+
+
+def _safe_frames(error: BaseException) -> list[str]:
+    lines = []
+    tb = _EXC_TRACEBACK(error)
+    while type(tb) is TracebackType:
+        code = tb.tb_frame.f_code
+        filename = code.co_filename
+        name = code.co_qualname
+        lineno = tb.tb_lineno
+        lines.append(
+            f'  File "{filename if type(filename) is str else "?"}", '
+            f"line {lineno if type(lineno) is int else '?'}, "
+            f"in {name if type(name) is str else '?'}\n"
+        )
+        tb = tb.tb_next
+    return lines
+
+
+def _safe_format_exception(error: BaseException) -> str:
+    """A traceback.format_exception look-alike that runs no exception code.
+
+    Frames come from the traceback objects (file, line, function; no source
+    lines, which would mean file I/O on the dying path); the message is the
+    exception's exact-``str`` args, never ``str(error)``. Chains follow
+    ``__cause__`` / ``__context__`` like the interpreter, bounded and
+    cycle-safe.
+    """
+    chain: list[tuple[BaseException, str]] = []
+    seen: set[int] = set()
+    current: BaseException | None = error
+    link = ""
+    while current is not None and len(chain) < _MAX_CHAIN:
+        if id(current) in seen:
+            break
+        seen.add(id(current))
+        chain.append((current, link))
+        cause = _EXC_CAUSE(current)
+        if cause is not None:
+            current = cause
+            link = (
+                "\nThe above exception was the direct cause of the "
+                "following exception:\n\n"
+            )
+        elif _EXC_SUPPRESS_CONTEXT(current) is not True:
+            current = _EXC_CONTEXT(current)
+            link = (
+                "\nDuring handling of the above exception, another "
+                "exception occurred:\n\n"
+            )
+        else:
+            current = None
+    out: list[str] = []
+    # Innermost first, as the interpreter prints; each outer exception's link
+    # text sits between it and the exception it chains to.
+    for index in range(len(chain) - 1, -1, -1):
+        exc, _ = chain[index]
+        out.append("Traceback (most recent call last):\n")
+        out.extend(_safe_frames(exc))
+        out.append(_safe_exception_line(exc) + "\n")
+        if index > 0:
+            out.append(chain[index][1])
+    return "".join(out)
+
 
 def ple_fail_closed(
     *,
@@ -88,9 +204,10 @@ def ple_fail_closed(
         # Three-arg form: it does not depend on being inside an active except
         # block, and this function is called from one frame further out than
         # the raise.
-        tb = "".join(
-            traceback.format_exception(type(error), error, error.__traceback__)
-        )
+        # _safe_format_exception, not traceback.format_exception: see the note
+        # above _EXC_ARGS -- the stdlib formatter calls str(error), which is
+        # exception-defined code and can block the dying process forever.
+        tb = _safe_format_exception(error)
         raw = tb.encode("utf-8", "backslashreplace")
         # Truncate from the END, not the start. format_exception puts the
         # frames first and the exception TYPE AND MESSAGE last, so a head-side