[Rebase] Row dump removed, lint, knob read-site census
The debug PLE row dump is deleted; ruff findings on fork lines cleared; a read-site census test pins which fork knobs are read and where. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
+30
-17
@@ -345,8 +345,10 @@ if TYPE_CHECKING:
|
||||
# catch-all branch in _get_ple_embedding_quant_method, which lived in the
|
||||
# half of models/qwen4_exp/nvidia/ple_layer.py that #54371 deleted. With no
|
||||
# reader it cannot change an artifact, so hashing it would only invalidate
|
||||
# caches for nothing; and on this uncompiled NVIDIA path (#55272) there may
|
||||
# be no AOT artifact to key at all. It gives NO capture protection either.
|
||||
# caches for nothing. (The AOT artifact is real on this tree: #55272
|
||||
# removed torch.compile from the NVIDIA Qwen4Exp path, the compile-profile guard restored it,
|
||||
# and the vLLM 0.30 rebase ran AOT cache-load boots -- so the reason is "no
|
||||
# reader", not "no artifact".) It gives NO capture protection either.
|
||||
# v0.30.0 selects the FP8 PLE method from the CONFIG KEY
|
||||
# text_config.ple_embedding_dtype == "float8_e4m3fn"
|
||||
# (ngram_embedding.py:174-175) -- see
|
||||
@@ -434,14 +436,12 @@ if TYPE_CHECKING:
|
||||
# cross-change knob table). Each mirrors its read site's
|
||||
# parse; if a read site changes, the entry here must be reconciled.
|
||||
# (Both read sites below are now live.)
|
||||
# *** INERT ON THE 0.30 BASE: NO READ SITE. ***
|
||||
# Its reader is the slot-ownership assertion code (an earlier commit), which edits
|
||||
# vllm/v1/ple_offload/connector.py -- a module that does not exist on
|
||||
# v0.30.0. Setting this on the 0.30 base does NOTHING, and NOTHING WARNS
|
||||
# (validate_environ only reports UNregistered names). Already an ignored
|
||||
# factor, correctly. Kept registered so the contract is recorded; it
|
||||
# becomes live again only together with the pull transport
|
||||
# (the PLE transport choice).
|
||||
# LIVE: read once at
|
||||
# import as envs.VLLM_E2_PLE_FLAG_CHECK by vllm/v1/ple_offload/connector.py
|
||||
# (the PLE pull-transport flag check, rank-gated, off by default). On
|
||||
# the 0.30 base it had NO read site: the PLE-diagnostics half
|
||||
# had not been relanded after the PLE pull-transport port restored connector.py. The
|
||||
# census in tests/test_envs.py now asserts the read site exists.
|
||||
# AOT: no. CAPTURE: no -- strictly host-side and outside capture.
|
||||
# off|warn|raise (1 = warn), normalised by _e2_diagnostic_mode like the
|
||||
# rest of the family. The mamba ownership check moved this gate off a plain
|
||||
@@ -492,6 +492,9 @@ if TYPE_CHECKING:
|
||||
VLLM_NIC_SELECTION_VARS: str = ""
|
||||
VLLM_ENABLE_HPC_OPS: bool = False
|
||||
MERLIN_TOPK_EXACT_COUNTERS_LOG: int = 0
|
||||
# Pre-tie-determinism fallback for the counter reader's slot count; runtime-only
|
||||
# (ignored factor), never consulted on this build. See the dict entry.
|
||||
MERLIN_TOPK_COUNTER_SLOTS: str | None = None
|
||||
#
|
||||
# --- determinism toggles -----------------------------------------
|
||||
# The three nondeterminism mechanisms (FULL_K, QSA_SORT, TIE_DET/TIE_RECENT).
|
||||
@@ -561,11 +564,13 @@ if TYPE_CHECKING:
|
||||
# The same restart on an overflowing DECODE row.
|
||||
MERLIN_TOPK_EXACT_DECODE: str | None = None
|
||||
# Deterministic placement and tie membership on the LARGE path.
|
||||
# Registered here ahead of its code, exactly as the four toggles above
|
||||
# were, so the full tree is complete; its read site is
|
||||
# det_large_enabled() in topk.cu on the large path. Setting it without that
|
||||
# code does nothing, which is why it is a passthrough and not a parsed
|
||||
# bool that would imply this tree can act on it.
|
||||
# LIVE on this tree since the v2.2 kernel port: the
|
||||
# read site is det_large_enabled() in csrc/libtorch_stable/topk.cu, a
|
||||
# std::getenv resolved once per process, and the resolved value reaches
|
||||
# the large-path radix_topk as a launch argument, so a cudagraph bakes it
|
||||
# in. CAPTURE: yes. AOT: not established (built into the extension, not by
|
||||
# the AOT compiler); hashed conservatively like the switches above. A raw
|
||||
# passthrough because the C++ getter is the single source of truth.
|
||||
MERLIN_TOPK_DET_LARGE: str | None = None
|
||||
#
|
||||
# The large-path FAULT INJECTORS. TEST
|
||||
@@ -611,8 +616,8 @@ if TYPE_CHECKING:
|
||||
# without first deleting it from the fence -- which means writing down its
|
||||
# AOT and capture identity against its real read site -- is the exact
|
||||
# failure the fork's env rule exists to prevent, and
|
||||
# tests/test_envs.py::test_unaudited_host_side_knob_stays_listed_until_it_is_classified
|
||||
# fails on it.
|
||||
# test_unaudited_host_side_knob_stays_listed_until_it_is_classified (in
|
||||
# tests/test_envs.py) fails on it.
|
||||
#
|
||||
# The fence holds NOTHING BUT the names, one per line. Prose belongs out
|
||||
# here: the gate reads the fence as a set of identifiers, and a knob name
|
||||
@@ -2661,6 +2666,11 @@ environment_variables: dict[str, Callable[[], Any]] = {
|
||||
"MERLIN_TOPK_EXACT_COUNTERS_LOG": lambda: int(
|
||||
os.getenv("MERLIN_TOPK_EXACT_COUNTERS_LOG", "0")
|
||||
),
|
||||
# Fallback slot count for the counter reader on a build WITHOUT the
|
||||
# persistent_topk_exact_counters_slots op (pre-tie-determinism). This tree registers
|
||||
# that op, so topk_counters.slot_count() never consults the variable here.
|
||||
# Raw passthrough: the reader's int(os.getenv(..., "5")) is the parser.
|
||||
"MERLIN_TOPK_COUNTER_SLOTS": lambda: os.getenv("MERLIN_TOPK_COUNTER_SLOTS", None),
|
||||
# Determinism toggles. See the declaration block above for the AOT /
|
||||
# capture classification and for why three of these are raw passthroughs.
|
||||
# All four are absent from ignored_factors on purpose, so all four are
|
||||
@@ -2898,6 +2908,9 @@ def compile_factors() -> dict[str, object]:
|
||||
# host-side timer and a snapshot op, and is read by no compiled code, so
|
||||
# hashing it would invalidate every compile cache for a log interval.
|
||||
"MERLIN_TOPK_EXACT_COUNTERS_LOG",
|
||||
# Host-side reader fallback (buffer length for a snapshot op outside
|
||||
# capture); dead on a build that answers the _slots query.
|
||||
"MERLIN_TOPK_COUNTER_SLOTS",
|
||||
# ------------------------------------------------------------------
|
||||
"FORCE_COLOR",
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ _QSA_SORT = envs.MERLIN_QSA_SORT
|
||||
|
||||
|
||||
def _qsa_canonicalise(block_indices: torch.Tensor) -> None:
|
||||
"""Sort selected block indices ascending, padding last. In place, shape preserved."""
|
||||
"""Sort selected block indices ascending, padding last. In place, same shape."""
|
||||
if not _QSA_SORT:
|
||||
return
|
||||
big = torch.iinfo(block_indices.dtype).max
|
||||
|
||||
@@ -11,9 +11,6 @@ from typing import Any
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
from vllm.v1.ple_offload import rowdump as _rowdump
|
||||
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||
from vllm.v1.ple_offload.progress import PleProgressMonitor
|
||||
import torch.nn as nn
|
||||
import zmq
|
||||
from cuda.bindings import driver as cuda_driver
|
||||
@@ -28,6 +25,8 @@ from vllm.model_executor.layers.ple_offload_layer import (
|
||||
flag_check_is_authoritative,
|
||||
stale_offload_flags,
|
||||
)
|
||||
from vllm.v1.ple_offload.fatal import ple_fail_closed
|
||||
from vllm.v1.ple_offload.progress import PleProgressMonitor
|
||||
from vllm.v1.ple_offload.protocol import (
|
||||
PleOffloadRegistration,
|
||||
PleOffloadRequest,
|
||||
@@ -284,7 +283,8 @@ class PleOffloadConnector:
|
||||
}
|
||||
if not layers:
|
||||
raise RuntimeError(
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no PleOffloadLayer"
|
||||
"VLLM_E2_PLE_PULL_TRANSPORT is enabled, but the model has no "
|
||||
"PleOffloadLayer"
|
||||
)
|
||||
|
||||
config = vllm_config.model_config.hf_text_config
|
||||
@@ -711,7 +711,6 @@ class PleOffloadConnector:
|
||||
f"PLE request count {num_reqs} exceeds connector capacity "
|
||||
f"{self._query_start_loc_buf.shape[0] - 1}"
|
||||
)
|
||||
self._rowdump_n = None if dummy_run else num_tokens
|
||||
if dummy_run:
|
||||
self._signal_dummy_outputs_impl(num_tokens)
|
||||
return
|
||||
@@ -913,15 +912,6 @@ class PleOffloadConnector:
|
||||
correlation_id=correlation_id,
|
||||
):
|
||||
raise RuntimeError("PLE test fault: exc_in_release")
|
||||
n = getattr(self, "_rowdump_n", None)
|
||||
self._rowdump_n = None
|
||||
if _rowdump.enabled() and n is not None and not torch.cuda.is_current_stream_capturing():
|
||||
torch.cuda.current_stream(self.device).synchronize()
|
||||
seq = getattr(self, "_rowdump_seq", 0)
|
||||
self._rowdump_seq = seq + 1
|
||||
for layer in self._layers.values():
|
||||
_rowdump.write(f"gpu-tp{self.tp_rank}", 0, self.tp_rank, seq,
|
||||
self._input_ids_source[:n], layer._gpu_output_buffer[:n])
|
||||
# Reset only after the consumer forward so the CPU worker cannot
|
||||
# overwrite an output that a GPU PLE placeholder may still read.
|
||||
stream = torch.cuda.current_stream(self.device)
|
||||
|
||||
@@ -54,11 +54,7 @@ def ple_fail_closed(
|
||||
detail_suffix = ""
|
||||
if trusted_error:
|
||||
args = BaseException.args.__get__(error, error_type)
|
||||
if (
|
||||
type(args) is tuple
|
||||
and len(args) == 1
|
||||
and type(args[0]) is str
|
||||
):
|
||||
if type(args) is tuple and len(args) == 1 and type(args[0]) is str:
|
||||
detail = args[0][:512].replace("\n", "\\n")
|
||||
detail_suffix = f":{detail}"
|
||||
message = (
|
||||
@@ -105,7 +101,8 @@ def ple_fail_closed(
|
||||
raw = raw[-8192:]
|
||||
os.write(
|
||||
2,
|
||||
b"PLE fatal traceback (tail) follows:\n" if clipped
|
||||
b"PLE fatal traceback (tail) follows:\n"
|
||||
if clipped
|
||||
else b"PLE fatal traceback follows:\n",
|
||||
)
|
||||
os.write(2, raw)
|
||||
@@ -118,9 +115,13 @@ def ple_fail_closed(
|
||||
# Still guarded, because os._exit below must run whatever happens: an
|
||||
# escape from this handler would leave the worker alive in the state
|
||||
# the fail-closed path exists to end.
|
||||
try:
|
||||
try: # noqa: SIM105
|
||||
os.write(2, b"PLE fatal: traceback unavailable\n")
|
||||
except BaseException:
|
||||
# SIM105 declined: an earlier revision made this helper call no
|
||||
# exception-defined code on the dying path. contextlib.suppress
|
||||
# runs a Python __exit__ between here and os._exit below; a bare
|
||||
# except keeps that window empty.
|
||||
pass
|
||||
os._exit(1)
|
||||
|
||||
@@ -143,9 +144,7 @@ def fail_ple_execute_model(func: _F) -> _F:
|
||||
connector = getattr(self, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
model_runner = getattr(self, "model_runner", None)
|
||||
connector = getattr(
|
||||
model_runner, "_ple_offload_connector", None
|
||||
)
|
||||
connector = getattr(model_runner, "_ple_offload_connector", None)
|
||||
if connector is None:
|
||||
raise
|
||||
connector._fail_closed(func.__name__, error)
|
||||
|
||||
@@ -20,11 +20,7 @@ _POLL_INTERVAL_S = 0.1
|
||||
|
||||
def _validate_deadline(value: float, name: str) -> float:
|
||||
value = float(value)
|
||||
if (
|
||||
not math.isfinite(value)
|
||||
or value <= 0
|
||||
or value > _MAX_PROGRESS_DEADLINE_S
|
||||
):
|
||||
if not math.isfinite(value) or value <= 0 or value > _MAX_PROGRESS_DEADLINE_S:
|
||||
raise ValueError(
|
||||
f"{name} must be finite, greater than zero, and no greater than "
|
||||
f"{_MAX_PROGRESS_DEADLINE_S:g} seconds"
|
||||
@@ -123,10 +119,7 @@ class PleProgressMonitor:
|
||||
active = self._submission
|
||||
if active is None or active.sequence != sequence:
|
||||
raise RuntimeError("PLE fault correlation has no active submission")
|
||||
return (
|
||||
f"sequence:{active.sequence}"
|
||||
f"@begin:{active.started:.9f}"
|
||||
)
|
||||
return f"sequence:{active.sequence}@begin:{active.started:.9f}"
|
||||
|
||||
def completion_is_pending(self, sequence: int) -> bool:
|
||||
"""Whether this serving sequence has a retained, incomplete event.
|
||||
@@ -149,9 +142,7 @@ class PleProgressMonitor:
|
||||
|
||||
def begin(self, *, startup: bool, num_tokens: int) -> int:
|
||||
now = time.monotonic()
|
||||
budget = (
|
||||
self._startup_deadline_s if startup else self._serving_deadline_s
|
||||
)
|
||||
budget = self._startup_deadline_s if startup else self._serving_deadline_s
|
||||
with self._condition:
|
||||
active = self._submission
|
||||
if active is not None:
|
||||
|
||||
@@ -1,41 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Debug-only PLE row dump for byte-level transport checks (off unless VLLM_PLE_ROW_DUMP=<dir>).
|
||||
|
||||
Each process appends framed records to <dir>/<tag>-pid<pid>.bin:
|
||||
b"PLER" | kind u8 (0 = GPU worker: rows its forward consumed; 1 = offload process: rows it published)
|
||||
| rank u8 | reserved u16 | seq u32 | n u32 | row_bytes u32 | ids int64[n] | rows uint8[n * row_bytes]
|
||||
"""
|
||||
|
||||
import os
|
||||
import struct
|
||||
import threading
|
||||
|
||||
import torch
|
||||
|
||||
_DIR = os.environ.get("VLLM_PLE_ROW_DUMP", "")
|
||||
_LOCK = threading.Lock()
|
||||
_FILES: dict = {}
|
||||
MAGIC = b"PLER"
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
return bool(_DIR)
|
||||
|
||||
|
||||
def write(tag: str, kind: int, rank: int, seq: int, ids: torch.Tensor, rows: torch.Tensor) -> None:
|
||||
ids_cpu = ids.detach().to(device="cpu", dtype=torch.int64).contiguous()
|
||||
rows_cpu = rows.detach().to(device="cpu").contiguous()
|
||||
n = int(ids_cpu.shape[0])
|
||||
if rows_cpu.shape[0] != n:
|
||||
raise RuntimeError(f"PLE row dump: {rows_cpu.shape[0]} rows for {n} ids")
|
||||
raw = rows_cpu.view(torch.uint8).numpy().tobytes() if n else b""
|
||||
row_bytes = len(raw) // n if n else 0
|
||||
record = (MAGIC + struct.pack("<BBHIII", kind, rank, 0, seq, n, row_bytes)
|
||||
+ ids_cpu.numpy().tobytes() + raw)
|
||||
with _LOCK:
|
||||
f = _FILES.get(tag)
|
||||
if f is None:
|
||||
os.makedirs(_DIR, exist_ok=True)
|
||||
f = open(os.path.join(_DIR, f"{tag}-pid{os.getpid()}.bin"), "ab", buffering=0)
|
||||
_FILES[tag] = f
|
||||
f.write(record)
|
||||
@@ -40,7 +40,6 @@ from typing import Any, cast
|
||||
|
||||
import msgspec
|
||||
import torch
|
||||
from vllm.v1.ple_offload import rowdump as _rowdump
|
||||
import torch.distributed as dist
|
||||
import zmq
|
||||
|
||||
@@ -453,26 +452,20 @@ class PleOffloadWorkerHandle:
|
||||
# Keep parent-death supervision connected while SIGTERM lets
|
||||
# proc_main run its shutdown event and finally cleanup.
|
||||
self.proc.terminate()
|
||||
try:
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
self.proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
if self.death_writer is not None:
|
||||
self.death_writer.close()
|
||||
self.death_writer = None
|
||||
if not graceful and self.proc.poll() is None:
|
||||
# Expired launch: retain the immediate EOF -> TERM path.
|
||||
self.proc.terminate()
|
||||
try:
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
self.proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
if self.proc.poll() is None:
|
||||
self.proc.kill()
|
||||
try:
|
||||
with contextlib.suppress(subprocess.TimeoutExpired):
|
||||
self.proc.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
pass
|
||||
# The child normally removes the endpoint in proc_main's finally.
|
||||
# Once it has exited, this exact PLE-owned address is safe to remove as
|
||||
# a fallback for SIGKILL or an interrupted child cleanup.
|
||||
@@ -1234,12 +1227,6 @@ class PleOffloadRunner:
|
||||
ngram_context,
|
||||
output_buffer=self._scratch_bufs[dp_rank][layer_name],
|
||||
)
|
||||
if _rowdump.enabled():
|
||||
_seq = getattr(self, "_rowdump_seq", 0)
|
||||
self._rowdump_seq = _seq + 1
|
||||
_rowdump.write(f"offload-dp{dp_rank}", 1, dp_rank, _seq,
|
||||
input_bufs.input_ids_buf[: request.num_tokens], result)
|
||||
|
||||
# The result is identical on every TP rank in this DP group.
|
||||
# A worker's host buffer may be overwritten only after its GPU
|
||||
# released the previous result (``release_outputs`` stores
|
||||
|
||||
+117
-53
@@ -1,40 +1,57 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Debug-only allocator telemetry for E2, BOUNDARY-ONLY. Off unless VLLM_E2_MEMSTATS=<ctl poll seconds> and
|
||||
VLLM_E2_MEMSTATS_DIR=<dir>.
|
||||
"""Debug-only allocator telemetry for E2, BOUNDARY-ONLY. Off unless
|
||||
VLLM_E2_MEMSTATS=<ctl poll seconds> and VLLM_E2_MEMSTATS_DIR=<dir>.
|
||||
|
||||
WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read torch.cuda.memory_stats() and
|
||||
reset_peak_memory_stats() every VLLM_E2_MEMSTATS seconds. Both take the per-device caching-allocator mutex and the
|
||||
Python bindings do not release the GIL, so the old contract's claim that this thread "takes no lock shared with the
|
||||
model path" was FALSE. Three one-boot runs at one KV pin showed a monotone dose-response: sampling at 4 Hz wedged a
|
||||
stage-0 worker during a 131072-token row; at 1.1 Hz it survived that row and wedged during a 260000-token row; with the
|
||||
thread disabled the run completed every row. Causation is not established at n=1 per run
|
||||
(single runs only), but polling has no place inside a measured region, and it was
|
||||
never necessary: PyTorch maintains the allocator high-water marks itself, so reading them once per window is exact.
|
||||
WHY THIS IS NOT A SAMPLER ANY MORE. An earlier version read
|
||||
torch.cuda.memory_stats() and reset_peak_memory_stats() every VLLM_E2_MEMSTATS
|
||||
seconds. Both take the per-device caching-allocator mutex and the Python
|
||||
bindings do not release the GIL, so the old contract's claim that this thread
|
||||
"takes no lock shared with the model path" was FALSE. Three one-boot arms at a
|
||||
4.1e9 KV pin showed a monotone dose-response: sampling at 4 Hz wedged a stage-0
|
||||
worker during a 131K thinking-on row; at 1.1 Hz it survived that row and wedged during
|
||||
a 260K row; with the thread disabled the run completed every row. Causation
|
||||
is not established at n=1 per arm (a known limit of
|
||||
that capacity probe), but polling has no place
|
||||
inside a measured region, and it was never necessary: PyTorch maintains the
|
||||
allocator high-water marks itself, so reading them once per window is exact.
|
||||
|
||||
WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and touches NO allocator state: its tick
|
||||
is a plain read of <dir>/window.ctl, which holds "<seq> <name>". The gate driver crosses a boundary only while the
|
||||
server is quiet (no request running). On a new seq the thread performs these boundary-only allocator reads:
|
||||
1. memory_stats() -> the closing window's exact high-water marks and cumulative retry/OOM counters
|
||||
2. reset_peak_memory_stats() -> immediately after the read, with no file I/O in the gap
|
||||
WHAT THIS THREAD DOES NOW. Between window boundaries it makes NO CUDA calls and
|
||||
touches NO allocator state: its tick is a plain read of <dir>/window.ctl, which
|
||||
holds "<seq> <name>". The gate driver crosses a boundary only while the server
|
||||
is quiet (no request running). On a new seq the thread performs these
|
||||
boundary-only allocator reads:
|
||||
1. memory_stats() -> the closing window's exact high-water marks
|
||||
and cumulative retry/OOM counters
|
||||
2. reset_peak_memory_stats() -> immediately after the read, with no file I/O
|
||||
in the gap
|
||||
3. mem_get_info() -> device free/total AT THAT QUIESCENT INSTANT
|
||||
4. host_memory_stats() -> pinned-host allocator current values, outside the read-to-reset gap
|
||||
then writes the "close" record for the old window and the "ack" record for the new one.
|
||||
4. host_memory_stats() -> pinned-host allocator current values, outside
|
||||
the read-to-reset gap
|
||||
then writes the "close" record for the old window and the "ack" record for the
|
||||
new one.
|
||||
|
||||
WHAT THIS MEASURES.
|
||||
- A row's allocator peak is EXACT for that row's window (not a sampled maximum; it cannot miss a transient).
|
||||
- Device free/total is observed ONLY at quiescent boundaries. No minimum-free figure exists for the inside of a
|
||||
row, and none may be reported.
|
||||
- Retry and OOM counters are cumulative and never reset; per-row deltas come from consecutive close records.
|
||||
- There is NO interval-level peak. A qualifying interval is narrower than its row and is identified only
|
||||
afterwards from the scheduler log, so the peak attributable to it is the ROW-level high-water mark, which
|
||||
over-includes the prefill ramp. This loss of resolution is inherent to the design.
|
||||
- A row's allocator peak is EXACT for that row's window (not a sampled
|
||||
maximum; it cannot miss a transient).
|
||||
- Device free/total is observed ONLY at quiescent boundaries. No minimum-free
|
||||
figure exists for the inside of a row, and none may be reported.
|
||||
- Retry and OOM counters are cumulative and never reset; per-row deltas come
|
||||
from consecutive close records.
|
||||
- There is NO interval-level peak. A qualifying interval is narrower
|
||||
than its row and is identified only afterwards from the scheduler log, so
|
||||
the peak attributable to it is the ROW-level high-water mark, which
|
||||
over-includes the prefill ramp. Report this loss of resolution
|
||||
explicitly.
|
||||
|
||||
Records (one JSON per line): kind start | close | ack | error; rank, local, dev, pid; window, seq; and for close:
|
||||
allocated_bytes.all.current, reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free, total,
|
||||
run_alloc_peak, run_res_peak, and a host object containing every host_memory_stats() key ending in ".current".
|
||||
Records are written with synchronous file I/O, never inside the read-to-reset gap. Nothing
|
||||
here raises into the worker.
|
||||
Records (one JSON per line): kind start | close | ack | error; rank, local,
|
||||
dev, pid; window, seq; and for close: allocated_bytes.all.current,
|
||||
reserved_bytes.all.current, alloc_peak_w, res_peak_w, retries, ooms, free,
|
||||
total, run_alloc_peak, run_res_peak, and a host object containing every
|
||||
host_memory_stats() key ending in ".current". Records are written with
|
||||
synchronous file I/O, never inside the read-to-reset gap. Nothing here raises
|
||||
into the worker.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import threading
|
||||
@@ -57,17 +74,11 @@ def _close_metrics(dev):
|
||||
free, total = torch.cuda.mem_get_info(dev)
|
||||
host_stats = torch.cuda.host_memory_stats()
|
||||
host = {
|
||||
key: int(value)
|
||||
for key, value in host_stats.items()
|
||||
if key.endswith(".current")
|
||||
key: int(value) for key, value in host_stats.items() if key.endswith(".current")
|
||||
}
|
||||
return {
|
||||
"allocated_bytes.all.current": int(
|
||||
s.get("allocated_bytes.all.current", 0)
|
||||
),
|
||||
"reserved_bytes.all.current": int(
|
||||
s.get("reserved_bytes.all.current", 0)
|
||||
),
|
||||
"allocated_bytes.all.current": int(s.get("allocated_bytes.all.current", 0)),
|
||||
"reserved_bytes.all.current": int(s.get("reserved_bytes.all.current", 0)),
|
||||
"alloc_peak_w": ap,
|
||||
"res_peak_w": rp,
|
||||
"retries": int(s.get("num_alloc_retries", 0)),
|
||||
@@ -85,9 +96,20 @@ def enabled() -> bool:
|
||||
def ensure_started(worker, armed_by: str = "compile_or_warm_up_model") -> None:
|
||||
if _started:
|
||||
return
|
||||
dev = torch.cuda.current_device() # captured on the worker's own thread: every call below passes it explicitly
|
||||
t = threading.Thread(target=_loop, name="e2-memstats", daemon=True,
|
||||
args=(dev, getattr(worker, "rank", -1), getattr(worker, "local_rank", -1), armed_by))
|
||||
dev = (
|
||||
torch.cuda.current_device()
|
||||
) # captured on the worker's own thread: every call below passes it explicitly
|
||||
t = threading.Thread(
|
||||
target=_loop,
|
||||
name="e2-memstats",
|
||||
daemon=True,
|
||||
args=(
|
||||
dev,
|
||||
getattr(worker, "rank", -1),
|
||||
getattr(worker, "local_rank", -1),
|
||||
armed_by,
|
||||
),
|
||||
)
|
||||
_started.append(t)
|
||||
t.start()
|
||||
|
||||
@@ -103,7 +125,13 @@ def _ctl():
|
||||
|
||||
def _loop(dev, rank, local, armed_by):
|
||||
os.makedirs(_DIR, exist_ok=True)
|
||||
f = open(os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"), "a", buffering=1)
|
||||
# noqa SIM115: the handle lives as long as the telemetry daemon thread
|
||||
# that writes to it; there is no scope to close it in.
|
||||
f = open( # noqa: SIM115
|
||||
os.path.join(_DIR, f"memstats-rank{rank}-pid{os.getpid()}.jsonl"),
|
||||
"a",
|
||||
buffering=1,
|
||||
)
|
||||
base = {"rank": rank, "local": local, "dev": dev, "pid": os.getpid()}
|
||||
|
||||
def w(rec):
|
||||
@@ -112,25 +140,61 @@ def _loop(dev, rank, local, armed_by):
|
||||
run_ap = run_rp = 0
|
||||
seq, name = _ctl() or (0, "serve")
|
||||
torch.cuda.reset_peak_memory_stats(dev)
|
||||
w({"kind": "start", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "armed_by": armed_by,
|
||||
"window": name, "seq": seq, "version": VERSION, "poll_s": _SEG})
|
||||
w(
|
||||
{
|
||||
"kind": "start",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"armed_by": armed_by,
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
"version": VERSION,
|
||||
"poll_s": _SEG,
|
||||
}
|
||||
)
|
||||
while True:
|
||||
time.sleep(_SEG)
|
||||
try:
|
||||
c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
|
||||
c = _ctl() # file read only: no CUDA, no allocator lock, no GIL-held call
|
||||
if c is None or c[0] == seq:
|
||||
continue
|
||||
# Boundary. The driver only crosses one while the server is quiet, so this is the single point at which
|
||||
# this thread touches the allocator at all.
|
||||
# Boundary. The driver only crosses one while the server is
|
||||
# quiet, so this is the single point at which this thread
|
||||
# touches the allocator at all.
|
||||
metrics = _close_metrics(dev)
|
||||
ap = metrics["alloc_peak_w"]
|
||||
rp = metrics["res_peak_w"]
|
||||
run_ap, run_rp = max(run_ap, ap), max(run_rp, rp)
|
||||
w({"kind": "close", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name,
|
||||
"seq": seq, **metrics,
|
||||
"run_alloc_peak": run_ap, "run_res_peak": run_rp})
|
||||
w(
|
||||
{
|
||||
"kind": "close",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
**metrics,
|
||||
"run_alloc_peak": run_ap,
|
||||
"run_res_peak": run_rp,
|
||||
}
|
||||
)
|
||||
seq, name = c
|
||||
w({"kind": "ack", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name, "seq": seq})
|
||||
w(
|
||||
{
|
||||
"kind": "ack",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
}
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 - telemetry must never raise into the worker
|
||||
w({"kind": "error", "t": round(time.time(), 4), "m": round(time.monotonic(), 4), "window": name,
|
||||
"seq": seq, "error": f"{type(e).__name__}: {e}"[:300]})
|
||||
w(
|
||||
{
|
||||
"kind": "error",
|
||||
"t": round(time.time(), 4),
|
||||
"m": round(time.monotonic(), 4),
|
||||
"window": name,
|
||||
"seq": seq,
|
||||
"error": f"{type(e).__name__}: {e}"[:300],
|
||||
}
|
||||
)
|
||||
|
||||
@@ -17,6 +17,8 @@ _FAULT = None
|
||||
if os.environ.get("VLLM_PLE_TEST_FAULT") is not None:
|
||||
from vllm_ple_test_fault import (
|
||||
_FAULT as _FAULT,
|
||||
)
|
||||
from vllm_ple_test_fault import (
|
||||
_block_on_harness_gate,
|
||||
)
|
||||
|
||||
@@ -160,9 +162,11 @@ def main() -> None:
|
||||
death_pipe = Connection(death_fd)
|
||||
|
||||
def monitor_parent() -> None:
|
||||
try:
|
||||
try: # noqa: SIM105
|
||||
death_pipe.recv()
|
||||
except (EOFError, OSError):
|
||||
# SIM105 declined: this module's contract is the docstring above --
|
||||
# its import list is the audited surface of the pre-torch child.
|
||||
pass
|
||||
os._exit(1)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user