diff --git a/vllm/v1/ple_offload/fatal.py b/vllm/v1/ple_offload/fatal.py index 90ab4a01ef..a96515f25d 100644 --- a/vllm/v1/ple_offload/fatal.py +++ b/vllm/v1/ple_offload/fatal.py @@ -5,8 +5,8 @@ import functools import os import time -import traceback from collections.abc import Callable +from types import TracebackType from typing import Any, NoReturn, TypeVar _F = TypeVar("_F", bound=Callable[..., Any]) @@ -26,6 +26,122 @@ _TRUSTED_ERROR_LABELS = ( (KeyboardInterrupt, "KeyboardInterrupt"), ) +# Slot readers for the dying path. Each is the C getter of the base type, +# called directly, so no __getattribute__, property, metaclass attribute or +# descriptor defined by the exception (or its class) can run. The traceback +# module is NOT safe here: traceback.format_exception calls str(error) (and +# str() on every chained exception) and reads error.__traceback__ through the +# instance's own __getattribute__, so an exception whose __str__ or +# __getattribute__ blocks kept the process alive forever -- the fail-closed +# path hung instead of dying (found by the blocking-exception transport tests). +_EXC_ARGS = BaseException.args.__get__ +_EXC_TRACEBACK = BaseException.__traceback__.__get__ +_EXC_CAUSE = BaseException.__cause__.__get__ +_EXC_CONTEXT = BaseException.__context__.__get__ +_EXC_SUPPRESS_CONTEXT = BaseException.__suppress_context__.__get__ +_TYPE_QUALNAME = type.__dict__["__qualname__"].__get__ +_TYPE_MODULE = type.__dict__["__module__"].__get__ +_MAX_CHAIN = 16 +_MAX_ARG_CHARS = 1024 + + +def _safe_type_label(cls: type) -> str: + """``module.qualname`` of ``cls`` without running class-defined code.""" + try: + qualname = _TYPE_QUALNAME(cls) + module = _TYPE_MODULE(cls) + except BaseException: + return "" + if type(qualname) is not str: + qualname = "" + if type(module) is str and module != "builtins": + return f"{module}.{qualname}" + return qualname + + +def _safe_exception_line(error: BaseException) -> str: + """Type label plus the exact-``str``/``int`` args; nothing else is shown.""" + label = _safe_type_label(type(error)) + args = _EXC_ARGS(error) + if type(args) is not tuple or len(args) == 0: + return label + parts = [] + for arg in args[:4]: + if type(arg) is str: + parts.append(arg[:_MAX_ARG_CHARS]) + elif type(arg) is int and -(10**18) < arg < 10**18: + parts.append(str(arg)) + else: + parts.append(f"<{_safe_type_label(type(arg))}>") + if len(args) > 4: + parts.append("...") + if len(args) == 1: + return f"{label}: {parts[0]}" + return f"{label}: ({', '.join(parts)})" + + +def _safe_frames(error: BaseException) -> list[str]: + lines = [] + tb = _EXC_TRACEBACK(error) + while type(tb) is TracebackType: + code = tb.tb_frame.f_code + filename = code.co_filename + name = code.co_qualname + lineno = tb.tb_lineno + lines.append( + f' File "{filename if type(filename) is str else "?"}", ' + f"line {lineno if type(lineno) is int else '?'}, " + f"in {name if type(name) is str else '?'}\n" + ) + tb = tb.tb_next + return lines + + +def _safe_format_exception(error: BaseException) -> str: + """A traceback.format_exception look-alike that runs no exception code. + + Frames come from the traceback objects (file, line, function; no source + lines, which would mean file I/O on the dying path); the message is the + exception's exact-``str`` args, never ``str(error)``. Chains follow + ``__cause__`` / ``__context__`` like the interpreter, bounded and + cycle-safe. + """ + chain: list[tuple[BaseException, str]] = [] + seen: set[int] = set() + current: BaseException | None = error + link = "" + while current is not None and len(chain) < _MAX_CHAIN: + if id(current) in seen: + break + seen.add(id(current)) + chain.append((current, link)) + cause = _EXC_CAUSE(current) + if cause is not None: + current = cause + link = ( + "\nThe above exception was the direct cause of the " + "following exception:\n\n" + ) + elif _EXC_SUPPRESS_CONTEXT(current) is not True: + current = _EXC_CONTEXT(current) + link = ( + "\nDuring handling of the above exception, another " + "exception occurred:\n\n" + ) + else: + current = None + out: list[str] = [] + # Innermost first, as the interpreter prints; each outer exception's link + # text sits between it and the exception it chains to. + for index in range(len(chain) - 1, -1, -1): + exc, _ = chain[index] + out.append("Traceback (most recent call last):\n") + out.extend(_safe_frames(exc)) + out.append(_safe_exception_line(exc) + "\n") + if index > 0: + out.append(chain[index][1]) + return "".join(out) + def ple_fail_closed( *, @@ -88,9 +204,10 @@ def ple_fail_closed( # Three-arg form: it does not depend on being inside an active except # block, and this function is called from one frame further out than # the raise. - tb = "".join( - traceback.format_exception(type(error), error, error.__traceback__) - ) + # _safe_format_exception, not traceback.format_exception: see the note + # above _EXC_ARGS -- the stdlib formatter calls str(error), which is + # exception-defined code and can block the dying process forever. + tb = _safe_format_exception(error) raw = tb.encode("utf-8", "backslashreplace") # Truncate from the END, not the start. format_exception puts the # frames first and the exception TYPE AND MESSAGE last, so a head-side