[Rebase] Row dump removed, lint, knob read-site census

The debug PLE row dump is deleted; ruff findings on fork lines cleared; a
read-site census test pins which fork knobs are read and where.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 20:01:43 +07:00
co-authored by Hinata
parent 7f6055caa2
commit e3fcd7e3ae
9 changed files with 172 additions and 165 deletions
+30 -17
View File
@@ -345,8 +345,10 @@ if TYPE_CHECKING:
# catch-all branch in _get_ple_embedding_quant_method, which lived in the
# half of models/qwen4_exp/nvidia/ple_layer.py that #54371 deleted. With no
# reader it cannot change an artifact, so hashing it would only invalidate
# caches for nothing; and on this uncompiled NVIDIA path (#55272) there may
# be no AOT artifact to key at all. It gives NO capture protection either.
# caches for nothing. (The AOT artifact is real on this tree: #55272
# removed torch.compile from the NVIDIA Qwen4Exp path, the compile-profile guard restored it,
# and the vLLM 0.30 rebase ran AOT cache-load boots -- so the reason is "no
# reader", not "no artifact".) It gives NO capture protection either.
# v0.30.0 selects the FP8 PLE method from the CONFIG KEY
# text_config.ple_embedding_dtype == "float8_e4m3fn"
# (ngram_embedding.py:174-175) -- see
@@ -434,14 +436,12 @@ if TYPE_CHECKING:
# cross-change knob table). Each mirrors its read site's
# parse; if a read site changes, the entry here must be reconciled.
# (Both read sites below are now live.)
# *** INERT ON THE 0.30 BASE: NO READ SITE. ***
# Its reader is the slot-ownership assertion code (an earlier commit), which edits
# vllm/v1/ple_offload/connector.py -- a module that does not exist on
# v0.30.0. Setting this on the 0.30 base does NOTHING, and NOTHING WARNS
# (validate_environ only reports UNregistered names). Already an ignored
# factor, correctly. Kept registered so the contract is recorded; it
# becomes live again only together with the pull transport
# (the PLE transport choice).
# LIVE: read once at
# import as envs.VLLM_E2_PLE_FLAG_CHECK by vllm/v1/ple_offload/connector.py
# (the PLE pull-transport flag check, rank-gated, off by default). On
# the 0.30 base it had NO read site: the PLE-diagnostics half
# had not been relanded after the PLE pull-transport port restored connector.py. The
# census in tests/test_envs.py now asserts the read site exists.
# AOT: no. CAPTURE: no -- strictly host-side and outside capture.
# off|warn|raise (1 = warn), normalised by _e2_diagnostic_mode like the
# rest of the family. The mamba ownership check moved this gate off a plain
@@ -492,6 +492,9 @@ if TYPE_CHECKING:
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_ENABLE_HPC_OPS: bool = False
MERLIN_TOPK_EXACT_COUNTERS_LOG: int = 0
# Pre-tie-determinism fallback for the counter reader's slot count; runtime-only
# (ignored factor), never consulted on this build. See the dict entry.
MERLIN_TOPK_COUNTER_SLOTS: str | None = None
#
# --- determinism toggles -----------------------------------------
# The three nondeterminism mechanisms (FULL_K, QSA_SORT, TIE_DET/TIE_RECENT).
@@ -561,11 +564,13 @@ if TYPE_CHECKING:
# The same restart on an overflowing DECODE row.
MERLIN_TOPK_EXACT_DECODE: str | None = None
# Deterministic placement and tie membership on the LARGE path.
# Registered here ahead of its code, exactly as the four toggles above
# were, so the full tree is complete; its read site is
# det_large_enabled() in topk.cu on the large path. Setting it without that
# code does nothing, which is why it is a passthrough and not a parsed
# bool that would imply this tree can act on it.
# LIVE on this tree since the v2.2 kernel port: the
# read site is det_large_enabled() in csrc/libtorch_stable/topk.cu, a
# std::getenv resolved once per process, and the resolved value reaches
# the large-path radix_topk as a launch argument, so a cudagraph bakes it
# in. CAPTURE: yes. AOT: not established (built into the extension, not by
# the AOT compiler); hashed conservatively like the switches above. A raw
# passthrough because the C++ getter is the single source of truth.
MERLIN_TOPK_DET_LARGE: str | None = None
#
# The large-path FAULT INJECTORS. TEST
@@ -611,8 +616,8 @@ if TYPE_CHECKING:
# without first deleting it from the fence -- which means writing down its
# AOT and capture identity against its real read site -- is the exact
# failure the fork's env rule exists to prevent, and
# tests/test_envs.py::test_unaudited_host_side_knob_stays_listed_until_it_is_classified
# fails on it.
# test_unaudited_host_side_knob_stays_listed_until_it_is_classified (in
# tests/test_envs.py) fails on it.
#
# The fence holds NOTHING BUT the names, one per line. Prose belongs out
# here: the gate reads the fence as a set of identifiers, and a knob name
@@ -2661,6 +2666,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
"MERLIN_TOPK_EXACT_COUNTERS_LOG": lambda: int(
os.getenv("MERLIN_TOPK_EXACT_COUNTERS_LOG", "0")
),
# Fallback slot count for the counter reader on a build WITHOUT the
# persistent_topk_exact_counters_slots op (pre-tie-determinism). This tree registers
# that op, so topk_counters.slot_count() never consults the variable here.
# Raw passthrough: the reader's int(os.getenv(..., "5")) is the parser.
"MERLIN_TOPK_COUNTER_SLOTS": lambda: os.getenv("MERLIN_TOPK_COUNTER_SLOTS", None),
# Determinism toggles. See the declaration block above for the AOT /
# capture classification and for why three of these are raw passthroughs.
# All four are absent from ignored_factors on purpose, so all four are
@@ -2898,6 +2908,9 @@ def compile_factors() -> dict[str, object]:
# host-side timer and a snapshot op, and is read by no compiled code, so
# hashing it would invalidate every compile cache for a log interval.
"MERLIN_TOPK_EXACT_COUNTERS_LOG",
# Host-side reader fallback (buffer length for a snapshot op outside
# capture); dead on a build that answers the _slots query.
"MERLIN_TOPK_COUNTER_SLOTS",
# ------------------------------------------------------------------
"FORCE_COLOR",
}
@@ -52,7 +52,7 @@ _QSA_SORT = envs.MERLIN_QSA_SORT
def _qsa_canonicalise(block_indices: torch.Tensor) -> None:
"""Sort selected block indices ascending, padding last. In place, shape preserved."""
"""Sort selected block indices ascending, padding last. In place, same shape."""
if not _QSA_SORT:
return
big = torch.iinfo(block_indices.dtype).max
+4 -14
View File
@@ -11,9 +11,6 @@ from typing import Any
import msgspec
import torch
from vllm.v1.ple_offload import rowdump as _rowdump
from vllm.v1.ple_offload.fatal import ple_fail_closed
from vllm.v1.ple_offload.progress import PleProgressMonitor
import torch.nn as nn
import zmq
from cuda.bindings import driver as cuda_driver
@@ -28,6 +25,8 @@ from vllm.model_executor.layers.ple_offload_layer import (
flag_check_is_authoritative,
stale_offload_flags,
)
from vllm.v1.ple_offload.fatal import ple_fail_closed
from vllm.v1.ple_offload.progress import PleProgressMonitor
from vllm.v1.ple_offload.protocol import (
PleOffloadRegistration,
PleOffloadRequest,
@@ -284,7 +283,8 @@ class PleOffloadConnector:
}
if not layers:
raise RuntimeError(
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no PleOffloadLayer"
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no "
"PleOffloadLayer"
)
config = vllm_config.model_config.hf_text_config
@@ -711,7 +711,6 @@ class PleOffloadConnector:
f"PLE request count {num_reqs} exceeds connector capacity "
f"{self._query_start_loc_buf.shape[0] - 1}"
)
self._rowdump_n = None if dummy_run else num_tokens
if dummy_run:
self._signal_dummy_outputs_impl(num_tokens)
return
@@ -913,15 +912,6 @@ class PleOffloadConnector:
correlation_id=correlation_id,
):
raise RuntimeError("PLE test fault: exc_in_release")
n = getattr(self, "_rowdump_n", None)
self._rowdump_n = None
if _rowdump.enabled() and n is not None and not torch.cuda.is_current_stream_capturing():
torch.cuda.current_stream(self.device).synchronize()
seq = getattr(self, "_rowdump_seq", 0)
self._rowdump_seq = seq + 1
for layer in self._layers.values():
_rowdump.write(f"gpu-tp{self.tp_rank}", 0, self.tp_rank, seq,
self._input_ids_source[:n], layer._gpu_output_buffer[:n])
# Reset only after the consumer forward so the CPU worker cannot
# overwrite an output that a GPU PLE placeholder may still read.
stream = torch.cuda.current_stream(self.device)
+9 -10
View File
@@ -54,11 +54,7 @@ def ple_fail_closed(
detail_suffix = ""
if trusted_error:
args = BaseException.args.__get__(error, error_type)
if (
type(args) is tuple
and len(args) == 1
and type(args[0]) is str
):
if type(args) is tuple and len(args) == 1 and type(args[0]) is str:
detail = args[0][:512].replace("\n", "\\n")
detail_suffix = f":{detail}"
message = (
@@ -105,7 +101,8 @@ def ple_fail_closed(
raw = raw[-8192:]
os.write(
2,
b"PLE fatal traceback (tail) follows:\n" if clipped
b"PLE fatal traceback (tail) follows:\n"
if clipped
else b"PLE fatal traceback follows:\n",
)
os.write(2, raw)
@@ -118,9 +115,13 @@ def ple_fail_closed(
# Still guarded, because os._exit below must run whatever happens: an
# escape from this handler would leave the worker alive in the state
# the fail-closed path exists to end.
try:
try: # noqa: SIM105
os.write(2, b"PLE fatal: traceback unavailable\n")
except BaseException:
# SIM105 declined: an earlier revision made this helper call no
# exception-defined code on the dying path. contextlib.suppress
# runs a Python __exit__ between here and os._exit below; a bare
# except keeps that window empty.
pass
os._exit(1)
@@ -143,9 +144,7 @@ def fail_ple_execute_model(func: _F) -> _F:
connector = getattr(self, "_ple_offload_connector", None)
if connector is None:
model_runner = getattr(self, "model_runner", None)
connector = getattr(
model_runner, "_ple_offload_connector", None
)
connector = getattr(model_runner, "_ple_offload_connector", None)
if connector is None:
raise
connector._fail_closed(func.__name__, error)
+3 -12
View File
@@ -20,11 +20,7 @@ _POLL_INTERVAL_S = 0.1
def _validate_deadline(value: float, name: str) -> float:
value = float(value)
if (
not math.isfinite(value)
or value <= 0
or value > _MAX_PROGRESS_DEADLINE_S
):
if not math.isfinite(value) or value <= 0 or value > _MAX_PROGRESS_DEADLINE_S:
raise ValueError(
f"{name} must be finite, greater than zero, and no greater than "
f"{_MAX_PROGRESS_DEADLINE_S:g} seconds"
@@ -123,10 +119,7 @@ class PleProgressMonitor:
active = self._submission
if active is None or active.sequence != sequence:
raise RuntimeError("PLE fault correlation has no active submission")
return (
f"sequence:{active.sequence}"
f"@begin:{active.started:.9f}"
)
return f"sequence:{active.sequence}@begin:{active.started:.9f}"
def completion_is_pending(self, sequence: int) -> bool:
"""Whether this serving sequence has a retained, incomplete event.
@@ -149,9 +142,7 @@ class PleProgressMonitor:
def begin(self, *, startup: bool, num_tokens: int) -> int:
now = time.monotonic()
budget = (
self._startup_deadline_s if startup else self._serving_deadline_s
)
budget = self._startup_deadline_s if startup else self._serving_deadline_s
with self._condition:
active = self._submission
if active is not None:
-41
View File
@@ -1,41 +0,0 @@
# SPDX-License-Identifier: Apache-2.0
"""Debug-only PLE row dump for byte-level transport checks (off unless VLLM_PLE_ROW_DUMP=<dir>).
Each process appends framed records to <dir>/<tag>-pid<pid>.bin:
b"PLER" | kind u8 (0 = GPU worker: rows its forward consumed; 1 = offload process: rows it published)
| rank u8 | reserved u16 | seq u32 | n u32 | row_bytes u32 | ids int64[n] | rows uint8[n * row_bytes]
"""
import os
import struct
import threading
import torch
_DIR = os.environ.get("VLLM_PLE_ROW_DUMP", "")
_LOCK = threading.Lock()
_FILES: dict = {}
MAGIC = b"PLER"
def enabled() -> bool:
return bool(_DIR)
def write(tag: str, kind: int, rank: int, seq: int, ids: torch.Tensor, rows: torch.Tensor) -> None:
ids_cpu = ids.detach().to(device="cpu", dtype=torch.int64).contiguous()
rows_cpu = rows.detach().to(device="cpu").contiguous()
n = int(ids_cpu.shape[0])
if rows_cpu.shape[0] != n:
raise RuntimeError(f"PLE row dump: {rows_cpu.shape[0]} rows for {n} ids")
raw = rows_cpu.view(torch.uint8).numpy().tobytes() if n else b""
row_bytes = len(raw) // n if n else 0
record = (MAGIC + struct.pack("<BBHIII", kind, rank, 0, seq, n, row_bytes)
+ ids_cpu.numpy().tobytes() + raw)
with _LOCK:
f = _FILES.get(tag)
if f is None:
os.makedirs(_DIR, exist_ok=True)
f = open(os.path.join(_DIR, f"{tag}-pid{os.getpid()}.bin"), "ab", buffering=0)
_FILES[tag] = f
f.write(record)
+3 -16
View File
@@ -40,7 +40,6 @@ from typing import Any, cast
import msgspec
import torch
from vllm.v1.ple_offload import rowdump as _rowdump
import torch.distributed as dist
import zmq
@@ -453,26 +452,20 @@ class PleOffloadWorkerHandle:
# Keep parent-death supervision connected while SIGTERM lets
# proc_main run its shutdown event and finally cleanup.
self.proc.terminate()
try:
with contextlib.suppress(subprocess.TimeoutExpired):
self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
pass
if self.death_writer is not None:
self.death_writer.close()
self.death_writer = None
if not graceful and self.proc.poll() is None:
# Expired launch: retain the immediate EOF -> TERM path.
self.proc.terminate()
try:
with contextlib.suppress(subprocess.TimeoutExpired):
self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
pass
if self.proc.poll() is None:
self.proc.kill()
try:
with contextlib.suppress(subprocess.TimeoutExpired):
self.proc.wait(timeout=5)
except subprocess.TimeoutExpired:
pass
# The child normally removes the endpoint in proc_main's finally.
# Once it has exited, this exact PLE-owned address is safe to remove as
# a fallback for SIGKILL or an interrupted child cleanup.
@@ -1234,12 +1227,6 @@ class PleOffloadRunner:
ngram_context,
output_buffer=self._scratch_bufs[dp_rank][layer_name],
)
if _rowdump.enabled():
_seq = getattr(self, "_rowdump_seq", 0)
self._rowdump_seq = _seq + 1
_rowdump.write(f"offload-dp{dp_rank}", 1, dp_rank, _seq,
input_bufs.input_ids_buf[: request.num_tokens], result)
# The result is identical on every TP rank in this DP group.
# A worker's host buffer may be overwritten only after its GPU
# released the previous result (``release_outputs`` stores
+117 -53
View File
@@ -1,40 +1,57 @@
# SPDX-License-Identifier: Apache-2.0
"""Debug-only allocator telemetry for E2, BOUNDARY-ONLY. Off unless VLLM_E2_MEMSTATS=<ctl poll seconds> and
VLLM_E2_MEMSTATS_DIR=<dir>.
"""Debug-only allocator telemetry for E2, BOUNDARY-ONLY. Off unless
VLLM_E2_MEMSTATS=<ctl poll seconds> and VLLM_E2_MEMSTATS_DIR=<dir>.
WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read torch.cuda.memory_stats() and
reset_peak_memory_stats() every VLLM_E2_MEMSTATS seconds. Both take the per-device caching-allocator mutex and the
Python bindings do not release the GIL, so the old contract's claim that this thread "takes no lock shared with the
model path" was FALSE. Three one-boot runs at one KV pin showed a monotone dose-response: sampling at 4 Hz wedged a
stage-0 worker during a 131072-token row; at 1.1 Hz it survived that row and wedged during a 260000-token row; with the
thread disabled the run completed every row. Causation is not established at n=1 per run
(single runs only), but polling has no place inside a measured region, and it was
never necessary: PyTorch maintains the allocator high-water marks itself, so reading them once per window is exact.
WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read
torch.cuda.memory_stats() and reset_peak_memory_stats() every VLLM_E2_MEMSTATS
seconds. Both take the per-device caching-allocator mutex and the Python
bindings do not release the GIL, so the old contract's claim that this thread
"takes no lock shared with the model path" was FALSE. Three one-boot arms at a
4.1e9 KV pin showed a monotone dose-response: sampling at 4 Hz wedged a stage-0
worker during a 131K thinking-on row; at 1.1 Hz it survived that row and wedged during
a 260K row; with the thread disabled the run completed every row. Causation
is not established at n=1 per arm (a known limit of
that capacity probe), but polling has no place
inside a measured region, and it was never necessary: PyTorch maintains the
allocator high-water marks itself, so reading them once per window is exact.
WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and touches NO allocator state: its tick
is a plain read of <dir>/window.ctl, which holds "<seq> <name>". The gate driver crosses a boundary only while the
server is quiet (no request running). On a new seq the thread performs these boundary-only allocator reads:
1. memory_stats() -> the closing window's exact high-water marks and cumulative retry/OOM counters
2. reset_peak_memory_stats() -> immediately after the read, with no file I/O in the gap
WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and
touches NO allocator state: its tick is a plain read of <dir>/window.ctl, which
holds "<seq> <name>". The gate driver crosses a boundary only while the server
is quiet (no request running). On a new seq the thread performs these
boundary-only allocator reads:
1. memory_stats() -> the closing window's exact high-water marks
and cumulative retry/OOM counters
2. reset_peak_memory_stats() -> immediately after the read, with no file I/O
in the gap
3. mem_get_info() -> device free/total AT THAT QUIESCENT INSTANT
4. host_memory_stats() -> pinned-host allocator current values, outside the read-to-reset gap
then writes the "close" record for the old window and the "ack" record for the new one.
4. host_memory_stats() -> pinned-host allocator current values, outside
the read-to-reset gap
then writes the "close" record for the old window and the "ack" record for the
new one.
WHAT THIS MEASURES.
- A row's allocator peak is EXACT for that row's window (not a sampled maximum; it cannot miss a transient).
- Device free/total is observed ONLY at quiescent boundaries. No minimum-free figure exists for the inside of a
row, and none may be reported.
- Retry and OOM counters are cumulative and never reset; per-row deltas come from consecutive close records.
- There is NO interval-level peak. A qualifying interval is narrower than its row and is identified only
afterwards from the scheduler log, so the peak attributable to it is the ROW-level high-water mark, which
over-includes the prefill ramp. This loss of resolution is inherent to the design.
- A row's allocator peak is EXACT for that row's window (not a sampled
maximum; it cannot miss a transient).
- Device free/total is observed ONLY at quiescent boundaries. No minimum-free
figure exists for the inside of a row, and none may be reported.
- Retry and OOM counters are cumulative and never reset; per-row deltas come
from consecutive close records.
- There is NO interval-level peak. A qualifying interval is narrower
than its row and is identified only afterwards from the scheduler log, so
the peak attributable to it is the ROW-level high-water mark, which
over-includes the prefill ramp. Report this loss of resolution
explicitly.
Records (one JSON per line): kind start | close | ack | error; rank, local, dev, pid; window, seq; and for close:
allocated_bytes.all.current, reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free, total,
run_alloc_peak, run_res_peak, and a host object containing every host_memory_stats() key ending in ".current".
Records are written with synchronous file I/O, never inside the read-to-reset gap. Nothing
here raises into the worker.
Records (one JSON per line): kind start | close | ack | error; rank, local,
dev, pid; window, seq; and for close: allocated_bytes.all.current,
reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free,
total, run_alloc_peak, run_res_peak, and a host object containing every
host_memory_stats() key ending in ".current". Records are written with
synchronous file I/O, never inside the read-to-reset gap. Nothing here raises
into the worker.
"""
import json
import os
import threading
@@ -57,17 +74,11 @@ def _close_metrics(dev):
free, total = torch.cuda.mem_get_info(dev)
host_stats = torch.cuda.host_memory_stats()
host = {
key: int(value)
for key, value in host_stats.items()
if key.endswith(".current")
key: int(value) for key, value in host_stats.items() if key.endswith(".current")
}
return {
"allocated_bytes.all.current": int(
s.get("allocated_bytes.all.current", 0)
),
"reserved_bytes.all.current": int(
s.get("reserved_bytes.all.current", 0)
),
"allocated_bytes.all.current": int(s.get("allocated_bytes.all.current", 0)),
"reserved_bytes.all.current": int(s.get("reserved_bytes.all.current", 0)),
"alloc_peak_w": ap,
"res_peak_w": rp,
"retries": int(s.get("num_alloc_retries", 0)),
@@ -85,9 +96,20 @@ def enabled() -> bool:
def ensure_started(worker, armed_by: str = "compile_or_warm_up_model") -> None:
if _started:
return
dev = torch.cuda.current_device() # captured on the worker's own thread: every call below passes it explicitly
t = threading.Thread(target=_loop, name="e2-memstats", daemon=True,
args=(dev, getattr(worker, "rank", -1), getattr(worker, "local_rank", -1), armed_by))
dev = (
torch.cuda.current_device()
) # captured on the worker's own thread: every call below passes it explicitly
t = threading.Thread(
target=_loop,
name="e2-memstats",
daemon=True,
args=(
dev,
getattr(worker, "rank", -1),
getattr(worker, "local_rank", -1),
armed_by,
),
)
_started.append(t)
t.start()
@@ -103,7 +125,13 @@ def _ctl():
def _loop(dev, rank, local, armed_by):
os.makedirs(_DIR, exist_ok=True)
f = open(os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"), "a", buffering=1)
# noqa SIM115: the handle lives as long as the telemetry daemon thread
# that writes to it; there is no scope to close it in.
f = open( # noqa: SIM115
os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"),
"a",
buffering=1,
)
base = {"rank": rank, "local": local, "dev": dev, "pid": os.getpid()}
def w(rec):
@@ -112,25 +140,61 @@ def _loop(dev, rank, local, armed_by):
run_ap = run_rp = 0
seq, name = _ctl() or (0, "serve")
torch.cuda.reset_peak_memory_stats(dev)
w({"kind": "start", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "armed_by": armed_by,
"window": name, "seq": seq, "version": VERSION, "poll_s": _SEG})
w(
{
"kind": "start",
"t": round(time.time(), 4),
"m": round(time.monotonic(), 4),
"armed_by": armed_by,
"window": name,
"seq": seq,
"version": VERSION,
"poll_s": _SEG,
}
)
while True:
time.sleep(_SEG)
try:
c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
if c is None or c[0] == seq:
continue
# Boundary. The driver only crosses one while the server is quiet, so this is the single point at which
# this thread touches the allocator at all.
# Boundary. The driver only crosses one while the server is
# quiet, so this is the single point at which this thread
# touches the allocator at all.
metrics = _close_metrics(dev)
ap = metrics["alloc_peak_w"]
rp = metrics["res_peak_w"]
run_ap, run_rp = max(run_ap, ap), max(run_rp, rp)
w({"kind": "close", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name,
"seq": seq, **metrics,
"run_alloc_peak": run_ap, "run_res_peak": run_rp})
w(
{
"kind": "close",
"t": round(time.time(), 4),
"m": round(time.monotonic(), 4),
"window": name,
"seq": seq,
**metrics,
"run_alloc_peak": run_ap,
"run_res_peak": run_rp,
}
)
seq, name = c
w({"kind": "ack", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name, "seq": seq})
w(
{
"kind": "ack",
"t": round(time.time(), 4),
"m": round(time.monotonic(), 4),
"window": name,
"seq": seq,
}
)
except Exception as e: # noqa: BLE001 - telemetry must never raise into the worker
w({"kind": "error", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name,
"seq": seq, "error": f"{type(e).__name__}: {e}"[:300]})
w(
{
"kind": "error",
"t": round(time.time(), 4),
"m": round(time.monotonic(), 4),
"window": name,
"seq": seq,
"error": f"{type(e).__name__}: {e}"[:300],
}
)
+5 -1
View File
@@ -17,6 +17,8 @@ _FAULT = None
if os.environ.get("VLLM_PLE_TEST_FAULT") is not None:
from vllm_ple_test_fault import (
_FAULT as _FAULT,
)
from vllm_ple_test_fault import (
_block_on_harness_gate,
)
@@ -160,9 +162,11 @@ def main() -> None:
death_pipe = Connection(death_fd)
def monitor_parent() -> None:
try:
try: # noqa: SIM105
death_pipe.recv()
except (EOFError, OSError):
# SIM105 declined: this module's contract is the docstring above --
# its import list is the audited surface of the pre-torch child.
pass
os._exit(1)