diff --git a/vllm/v1/ple_offload/rowdump.py b/vllm/v1/ple_offload/rowdump.py
new file mode 100644
index 0000000000..95fd3081b6
--- /dev/null
+++ b/vllm/v1/ple_offload/rowdump.py
@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Debug-only PLE row dump for byte-level transport checks (off unless VLLM_PLE_ROW_DUMP=
).
+
+Each process appends framed records to /-pid.bin:
+ b"PLER" | kind u8 (0 = GPU worker: rows its forward consumed; 1 = offload process: rows it published)
+ | rank u8 | reserved u16 | seq u32 | n u32 | row_bytes u32 | ids int64[n] | rows uint8[n * row_bytes]
+"""
+
+import os
+import struct
+import threading
+
+import torch
+
+_DIR = os.environ.get("VLLM_PLE_ROW_DUMP", "")
+_LOCK = threading.Lock()
+_FILES: dict = {}
+MAGIC = b"PLER"
+
+
+def enabled() -> bool:
+ return bool(_DIR)
+
+
+def write(tag: str, kind: int, rank: int, seq: int, ids: torch.Tensor, rows: torch.Tensor) -> None:
+ ids_cpu = ids.detach().to(device="cpu", dtype=torch.int64).contiguous()
+ rows_cpu = rows.detach().to(device="cpu").contiguous()
+ n = int(ids_cpu.shape[0])
+ if rows_cpu.shape[0] != n:
+ raise RuntimeError(f"PLE row dump: {rows_cpu.shape[0]} rows for {n} ids")
+ raw = rows_cpu.view(torch.uint8).numpy().tobytes() if n else b""
+ row_bytes = len(raw) // n if n else 0
+ record = (MAGIC + struct.pack(" and
+VLLM_E2_MEMSTATS_DIR=.
+
+WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read torch.cuda.memory_stats() and
+reset_peak_memory_stats() every VLLM_E2_MEMSTATS seconds. Both take the per-device caching-allocator mutex and the
+Python bindings do not release the GIL, so the old contract's claim that this thread "takes no lock shared with the
+model path" was FALSE. Three one-boot runs at one KV pin showed a monotone dose-response: sampling at 4 Hz wedged a
+stage-0 worker during a 131072-token row; at 1.1 Hz it survived that row and wedged during a 260000-token row; with the
+thread disabled the run completed every row. Causation is not established at n=1 per run
+(single runs only), but polling has no place inside a measured region, and it was
+never necessary: PyTorch maintains the allocator high-water marks itself, so reading them once per window is exact.
+
+WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and touches NO allocator state: its tick
+is a plain read of /window.ctl, which holds " ". The gate driver crosses a boundary only while the
+server is quiet (no request running). On a new seq the thread performs these boundary-only allocator reads:
+ 1. memory_stats() -> the closing window's exact high-water marks and cumulative retry/OOM counters
+ 2. reset_peak_memory_stats() -> immediately after the read, with no file I/O in the gap
+ 3. mem_get_info() -> device free/total AT THAT QUIESCENT INSTANT
+ 4. host_memory_stats() -> pinned-host allocator current values, outside the read-to-reset gap
+then writes the "close" record for the old window and the "ack" record for the new one.
+
+WHAT THIS MEASURES.
+ - A row's allocator peak is EXACT for that row's window (not a sampled maximum; it cannot miss a transient).
+ - Device free/total is observed ONLY at quiescent boundaries. No minimum-free figure exists for the inside of a
+ row, and none may be reported.
+ - Retry and OOM counters are cumulative and never reset; per-row deltas come from consecutive close records.
+ - There is NO interval-level peak. A qualifying interval is narrower than its row and is identified only
+ afterwards from the scheduler log, so the peak attributable to it is the ROW-level high-water mark, which
+ over-includes the prefill ramp. This loss of resolution is inherent to the design.
+
+Records (one JSON per line): kind start | close | ack | error; rank, local, dev, pid; window, seq; and for close:
+allocated_bytes.all.current, reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free, total,
+run_alloc_peak, run_res_peak, and a host object containing every host_memory_stats() key ending in ".current".
+Records are written with synchronous file I/O, never inside the read-to-reset gap. Nothing
+here raises into the worker.
+"""
+import json
+import os
+import threading
+import time
+
+import torch
+
+_SEG = float(os.environ.get("VLLM_E2_MEMSTATS", "0") or 0)
+_DIR = os.environ.get("VLLM_E2_MEMSTATS_DIR", "")
+_started = []
+VERSION = "e2_memstats/5-boundary-current-host"
+
+
+def _close_metrics(dev):
+ """Read one quiescent boundary; kept separate for CPU-only testing."""
+ s = torch.cuda.memory_stats(dev)
+ ap = int(s.get("allocated_bytes.all.peak", 0))
+ rp = int(s.get("reserved_bytes.all.peak", 0))
+ torch.cuda.reset_peak_memory_stats(dev) # no file I/O in the gap
+ free, total = torch.cuda.mem_get_info(dev)
+ host_stats = torch.cuda.host_memory_stats()
+ host = {
+ key: int(value)
+ for key, value in host_stats.items()
+ if key.endswith(".current")
+ }
+ return {
+ "allocated_bytes.all.current": int(
+ s.get("allocated_bytes.all.current", 0)
+ ),
+ "reserved_bytes.all.current": int(
+ s.get("reserved_bytes.all.current", 0)
+ ),
+ "alloc_peak_w": ap,
+ "res_peak_w": rp,
+ "retries": int(s.get("num_alloc_retries", 0)),
+ "ooms": int(s.get("num_ooms", 0)),
+ "free": int(free),
+ "total": int(total),
+ "host": host,
+ }
+
+
+def enabled() -> bool:
+ return _SEG > 0 and bool(_DIR)
+
+
+def ensure_started(worker, armed_by: str = "compile_or_warm_up_model") -> None:
+ if _started:
+ return
+ dev = torch.cuda.current_device() # captured on the worker's own thread: every call below passes it explicitly
+ t = threading.Thread(target=_loop, name="e2-memstats", daemon=True,
+ args=(dev, getattr(worker, "rank", -1), getattr(worker, "local_rank", -1), armed_by))
+ _started.append(t)
+ t.start()
+
+
+def _ctl():
+ try:
+ with open(os.path.join(_DIR, "window.ctl")) as f:
+ seq, name = f.read().split(None, 1)
+ return int(seq), name.strip()
+ except (OSError, ValueError):
+ return None
+
+
+def _loop(dev, rank, local, armed_by):
+ os.makedirs(_DIR, exist_ok=True)
+ f = open(os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"), "a", buffering=1)
+ base = {"rank": rank, "local": local, "dev": dev, "pid": os.getpid()}
+
+ def w(rec):
+ f.write(json.dumps({**base, **rec}, separators=(",", ":")) + "\n")
+
+ run_ap = run_rp = 0
+ seq, name = _ctl() or (0, "serve")
+ torch.cuda.reset_peak_memory_stats(dev)
+ w({"kind": "start", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "armed_by": armed_by,
+ "window": name, "seq": seq, "version": VERSION, "poll_s": _SEG})
+ while True:
+ time.sleep(_SEG)
+ try:
+ c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
+ if c is None or c[0] == seq:
+ continue
+ # Boundary. The driver only crosses one while the server is quiet, so this is the single point at which
+ # this thread touches the allocator at all.
+ metrics = _close_metrics(dev)
+ ap = metrics["alloc_peak_w"]
+ rp = metrics["res_peak_w"]
+ run_ap, run_rp = max(run_ap, ap), max(run_rp, rp)
+ w({"kind": "close", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name,
+ "seq": seq, **metrics,
+ "run_alloc_peak": run_ap, "run_res_peak": run_rp})
+ seq, name = c
+ w({"kind": "ack", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name, "seq": seq})
+ except Exception as e: # noqa: BLE001 - telemetry must never raise into the worker
+ w({"kind": "error", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name,
+ "seq": seq, "error": f"{type(e).__name__}: {e}"[:300]})
diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py
index 00d5806278..a90501415f 100644
--- a/vllm/v1/worker/gpu_worker.py
+++ b/vllm/v1/worker/gpu_worker.py
@@ -93,6 +93,7 @@ from vllm.v1.outputs import (
)
from vllm.v1.ple_offload.fatal import fail_ple_execute_model
from vllm.v1.utils import compute_iteration_details, report_usage_stats
+from vllm.v1.worker import e2_memstats as _e2_memstats
from vllm.v1.worker import guard_telemetry
from vllm.v1.worker.sentinel.gpu_worker_sentinel import WorkerSentinel
from vllm.v1.worker.startup_plan import (
@@ -1195,6 +1196,10 @@ class Worker(WorkerBase):
# while a capture is active is what cudaStreamCaptureModeGlobal rejects.
guard_telemetry.start_reader()
+ # Allocator telemetry: armed only after warm-up and CUDA-graph capture (env-gated).
+ if _e2_memstats.enabled():
+ _e2_memstats.ensure_started(self)
+
return CompilationTimes(
language_model=self.compilation_config.compilation_time,
encoder=self.compilation_config.encoder_compilation_time,