[Worker] E2-identity and rank-tagged e2-toggles startup lines

Each worker logs its launch identity and, after warm-up and capture, one
e2-toggles line with the effective toggle state read back from the C++
extensions through two new tensor-free query ops. Both extensions must be
rebuilt.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:56:00 +07:00
co-authored by Hinata
parent bef464a69d
commit 281491646d
8 changed files with 734 additions and 7 deletions
@@ -33,6 +33,7 @@
#include <torch/headeronly/util/Exception.h>
#include "libtorch_stable/torch_utils.h"
#include <atomic>
#include <cstdlib>
namespace {
@@ -53,14 +54,31 @@ namespace {
// This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw
// passthrough entry for validate_environ and the compile-factor hash only;
// nothing in Python may parse this name a second time.
//
// The initialiser also
// publishes the value it cached into g_cached_full_k (-1 = not initialised
// yet; nothing else writes it), and marlin_moe_full_k_state() below only
// LOADS it -- asking never fills the cache. Host code only.
// -------------------------------------------------------------------------
std::atomic<int> g_cached_full_k{-1};
bool merlin_full_k() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_FULL_K");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
const bool value =
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
g_cached_full_k.store(value ? 1 : 0, std::memory_order_release);
return value;
}();
return enabled;
}
// MERLIN_FULL_K as this process cached it: -1 = not initialised yet (no Marlin
// MoE launch has happened in this process), 0 = off, 1 = on. Tensor-free, no
// CUDA call.
int64_t marlin_moe_full_k_state() {
return static_cast<int64_t>(g_cached_full_k.load(std::memory_order_acquire));
}
} // namespace
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
@@ -736,3 +754,14 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
}
// v2.2 -- defined and registered in this TU so the query exists exactly when
// the static it reads does. Tensor-free, so CompositeExplicitAutograd (a CUDA
// registration could never be dispatched to without a tensor argument).
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
m.def("marlin_moe_full_k_state() -> int");
}
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
m.impl("marlin_moe_full_k_state", TORCH_BOX(&marlin_moe_full_k_state));
}
+4
View File
@@ -489,6 +489,10 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
// reader never has to guess it from an environment variable.
int64_t persistent_topk_exact_counters_slots();
// The persistent_topk switches as this
// process cached them, packed; see topk.cu for the layout.
int64_t persistent_topk_toggle_state();
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
void cooperative_topk(const torch::stable::Tensor& logits,
const torch::stable::Tensor& lengths,
+84 -5
View File
@@ -44,6 +44,29 @@ namespace {
// This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw
// passthrough for validate_environ and the compile-factor hash only.
// -------------------------------------------------------------------------
// -------------------------------------------------------------------------
// A READ-ONLY MIRROR of the
// switches as this process CACHED them, for persistent_topk_toggle_state().
//
// Each switch below is a function-local static filled once by its lambda. The
// lambda now also stores the value it cached into the matching atomic here
// (publish_cached), and nothing else ever writes these atomics, so a mirror
// either says "not initialised yet" (-1) or holds exactly the cached value.
// The query only LOADS them: it never calls a getter, so asking cannot fill a
// cache from an environment that differs from the one the kernel saw. The
// parse expressions are unchanged; this is host code only.
// -------------------------------------------------------------------------
std::atomic<int> g_cached_tie_recent{-1};
std::atomic<int> g_cached_tie_det{-1};
std::atomic<int> g_cached_exact_medium{-1};
std::atomic<int> g_cached_det_large{-1};
std::atomic<int> g_cached_exact_decode{-1};
bool publish_cached(std::atomic<int>& mirror, bool value) {
mirror.store(value ? 1 : 0, std::memory_order_release);
return value;
}
// MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above.
// Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode,
// most-recent-wins is +1.5 %. Both deterministic. Also the direction
@@ -53,7 +76,9 @@ namespace {
bool tie_recent_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TIE_RECENT");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
return publish_cached(
g_cached_tie_recent,
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
}();
return enabled;
}
@@ -61,7 +86,9 @@ bool tie_recent_enabled() {
bool det_ties_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TIE_DET");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
return publish_cached(
g_cached_tie_det,
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
}();
return enabled;
}
@@ -69,7 +96,9 @@ bool det_ties_enabled() {
bool exact_medium_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
return publish_cached(
g_cached_exact_medium,
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
}();
return enabled;
}
@@ -87,7 +116,9 @@ bool exact_medium_enabled() {
bool det_large_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_DET_LARGE");
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
return publish_cached(
g_cached_det_large,
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
}();
return enabled;
}
@@ -104,7 +135,9 @@ bool det_large_enabled() {
bool exact_decode_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
return publish_cached(
g_cached_exact_decode,
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
}();
return enabled;
}
@@ -887,3 +920,49 @@ int64_t persistent_topk_exact_counters_slots() {
return 0;
#endif
}
// The persistent_topk
// switches AS THIS PROCESS CACHED THEM -- the values every launch of this
// process uses -- read from the mirrors above. Tensor-free, no CUDA call, no
// allocation, never initialises a cache (see publish_cached), so it is safe
// from anywhere and asking changes nothing.
//
// Packed into one int64 so the op needs no tensor and no dispatch key.
// 2 bits per field, field i at bits [2i, 2i+1]: 0 = off, 1 = on,
// 2 = the static has not been initialised in this process yet.
// field 0 MERLIN_TIE_DET (det_ties_enabled)
// field 1 MERLIN_TIE_RECENT as parsed (tie_recent_enabled)
// field 2 effective direction, 1 = most-recent (effective_tie_recent)
// field 3 MERLIN_TOPK_EXACT_MEDIUM (exact_medium_enabled)
// field 4 MERLIN_TOPK_EXACT_DECODE (exact_decode_enabled)
// field 5 MERLIN_TOPK_DET_LARGE (det_large_enabled)
// bits [48, 55] = field count (6), bits [56, 62] = layout version (1). A
// reader must check both and refuse any other layout rather than guess.
int64_t persistent_topk_toggle_state() {
#ifndef USE_ROCM
const auto field = [](int cached) -> int64_t {
return cached < 0 ? 2 : (cached != 0 ? 1 : 0);
};
const int det = g_cached_tie_det.load(std::memory_order_acquire);
const int recent = g_cached_tie_recent.load(std::memory_order_acquire);
// The kernel's own rule, from the kernel's own header, on the cached values.
const int effective =
(det < 0 || recent < 0)
? -1
: (vllm::persistent::effective_tie_recent(det != 0, recent != 0)
? 1
: 0);
int64_t state = 0;
state |= field(det) << 0;
state |= field(recent) << 2;
state |= field(effective) << 4;
state |= field(g_cached_exact_medium.load(std::memory_order_acquire)) << 6;
state |= field(g_cached_exact_decode.load(std::memory_order_acquire)) << 8;
state |= field(g_cached_det_large.load(std::memory_order_acquire)) << 10;
state |= int64_t{6} << 48;
state |= int64_t{1} << 56;
return state;
#else
return 0; // layout version 0: no persistent_topk on ROCm
#endif
}
+5
View File
@@ -590,6 +590,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
ops.def(
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
ops.def("persistent_topk_exact_counters_slots() -> int");
// v2.2 -- the switches as this process cached them (topk.cu for the layout).
ops.def("persistent_topk_toggle_state() -> int");
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
ops.def(
@@ -947,6 +949,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
// NotImplementedError "no fallback function is registered").
ops.impl("persistent_topk_exact_counters_slots",
TORCH_BOX(&persistent_topk_exact_counters_slots));
// Tensor-free for the same reason as the slot count above.
ops.impl("persistent_topk_toggle_state",
TORCH_BOX(&persistent_topk_toggle_state));
}
// Cache ops
+148
View File
@@ -0,0 +1,148 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-rank launch-identity log line (log-only; changes no behaviour).
After model loading -- and after the PLE offload placeholders are armed -- every
GPU worker rank logs one line per model role it holds::
e2-identity[PP<p>_TP<t>]: ple_backend=<class>@<transport> ple_method=<class>
ple_weight_dtype=<dtype> breakable=<ON|OFF> compile_mode=<mode>
cudagraph_mode=<mode> role=<target|draft>
(one line in the log). ``role=draft`` is logged only where a draft model is
loaded (the last PP stage). Every value is read from the live objects:
* ``ple_backend``: the class of each PLE layer's ``ple_embedding`` module, and
the transport that serves its rows on this rank, read from the placeholder's
armed state -- ``host-mapped-pull`` (``_is_cpu_offloaded`` with a host output
pointer), ``ipc-push`` (offloaded, no host pointer) or ``local:<device>`` (the
rank owns the table).
* ``ple_method``: the embedding's live ``quant_method``, or, on an offloaded
placeholder, the ``_offload_quant_method`` the rank dequantizes with.
* ``ple_weight_dtype``: the table's dtype where the rank owns it; on an
offloaded placeholder, the dtype of the armed transfer buffer, into which the
offload process copies the table's rows unchanged
(``get_offload_output_dtype``).
* ``breakable``: ``is_breakable_cudagraph_enabled()``, the switch the capture
path itself reads.
* ``compile_mode`` / ``cudagraph_mode``: the role's ``CompilationConfig``
(config level, i.e. before KV-cache-time capture-mode resolution).
A role with no PLE layer reports ``none`` for the three PLE fields; several
distinct values on one rank are joined with ``,``.
"""
from typing import Any
from torch import nn
from vllm.logger import init_logger
logger = init_logger(__name__)
_NONE = "none"
def _join(values: set[str]) -> str:
return ",".join(sorted(values)) if values else _NONE
def _enum_name(value: Any) -> str:
if value is None:
return _NONE
return str(getattr(value, "name", value))
def _ple_embeddings(model: nn.Module | None) -> list[nn.Module]:
"""The ``ple_embedding`` module of every PLE layer in ``model``."""
if model is None:
return []
found = []
for module in model.modules():
embedding = module.__dict__.get("_modules", {}).get("ple_embedding")
if isinstance(embedding, nn.Module):
found.append(embedding)
return found
def _transport(embedding: nn.Module) -> str:
if getattr(embedding, "_is_cpu_offloaded", False):
if getattr(embedding, "_host_output_ptr", None) is not None:
return "host-mapped-pull"
return "ipc-push"
weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None)
if weight is not None:
return f"local:{weight.device.type}"
return "local:unknown"
def _method(embedding: nn.Module) -> str:
table = getattr(embedding, "ngram_embedding", None)
method = getattr(table, "quant_method", None)
if method is None:
method = getattr(embedding, "_offload_quant_method", None)
return type(method).__name__ if method is not None else _NONE
def _weight_dtype(embedding: nn.Module) -> str:
weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None)
if weight is not None:
return str(weight.dtype)
if getattr(embedding, "_is_cpu_offloaded", False):
buffer = getattr(embedding, "_gpu_output_buffer", None)
if buffer is not None:
return str(buffer.dtype)
return "unknown"
def _rank_tag() -> str:
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
def _draft(model_runner: Any) -> tuple[nn.Module | None, Any]:
"""The loaded draft model and its config (V2 speculator or V1 drafter)."""
for name in ("speculator", "drafter"):
holder = getattr(model_runner, name, None)
model = getattr(holder, "model", None)
if isinstance(model, nn.Module):
return model, getattr(holder, "vllm_config", None)
return None, None
def identity_line(
tag: str, model: nn.Module | None, vllm_config: Any, role: str
) -> str:
from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled
embeddings = _ple_embeddings(model)
backends = {f"{type(e).__name__}@{_transport(e)}" for e in embeddings}
compilation = getattr(vllm_config, "compilation_config", None)
return (
f"e2-identity[{tag}]: "
f"ple_backend={_join(backends)} "
f"ple_method={_join({_method(e) for e in embeddings})} "
f"ple_weight_dtype={_join({_weight_dtype(e) for e in embeddings})} "
f"breakable={'ON' if is_breakable_cudagraph_enabled() else 'OFF'} "
f"compile_mode={_enum_name(getattr(compilation, 'mode', None))} "
f"cudagraph_mode={_enum_name(getattr(compilation, 'cudagraph_mode', None))} "
f"role={role}"
)
def log_launch_identity(model_runner: Any, vllm_config: Any) -> None:
"""Log the identity line(s) for this rank. Never raises."""
tag = "PP?_TP?"
try:
tag = _rank_tag()
roles = [("target", model_runner.get_model(), vllm_config)]
draft_model, draft_config = _draft(model_runner)
if draft_model is not None:
roles.append(("draft", draft_model, draft_config or vllm_config))
for role, model, config in roles:
logger.info("%s", identity_line(tag, model, config, role))
except Exception as error: # log-only: identity must never break a boot
logger.warning(
"e2-identity[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
)
+444
View File
@@ -0,0 +1,444 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Per-rank determinism-toggle line (log-only; changes no behaviour).
Why this exists: the effective state of the
four determinism toggles and the three persistent_topk switches must be
attributable to a RANK and a PID. The kernel's own stderr line
(``[merlin] persistent_topk tie policy: ...``, topk.cu report_tie_policy_once)
is printed once per process with no tag, and FULL_K / QSA_SORT have no log
producer at all. Every GPU worker therefore logs exactly one line::
e2-toggles[PP<p>_TP<t>]: pid=<pid> MERLIN_FULL_K=<raw> MERLIN_QSA_SORT=<raw>
MERLIN_TIE_DET=<raw> MERLIN_TIE_RECENT=<raw> MERLIN_TOPK_EXACT_MEDIUM=<raw>
MERLIN_TOPK_EXACT_DECODE=<raw> MERLIN_TOPK_DET_LARGE=<raw>
effective: full_k=<0|1> qsa_sort=<0|1|invalid> tie_det=<0|1> tie_recent=<0|1>
tie_recent_effective=<0|1> exact_medium=<0|1> exact_decode=<0|1>
det_large=<0|1> source=<libc-getenv|os-environ>
cpp-cache: <full_k=... tie_det=... tie_recent=... tie_recent_effective=...
exact_medium=... exact_decode=... det_large=... source=cpp-cache
| n-a reason=<why>>
python: qsa_sort=<0|1|n-a> source=qsa_indexer._QSA_SORT
device: switch=<0|1|n-a> switch_large=<0|1|n-a> source=counter-slots-8-9
(one line in the log). ``<raw>`` is ``unset``, a plain token, or a double-quoted
string with every byte outside ``[!-~]`` (and ``"`` / ``\\``) hex-escaped, so
the line always splits on whitespace.
WHAT EACH SECTION IS
* ``effective:`` -- the environment at emission time, parsed with the SAME
rules the readers use (C++ ``std::getenv`` + ``std::atoi != 0`` for the six
C++ names, ``bool(int(...))`` for QSA_SORT; file:line of each below). The
C++ names are read through libc ``getenv`` -- the ``environ`` that
``std::getenv`` reads -- not ``os.environ``, which can diverge from it after
an ``os.putenv``. A MIRROR of the parse, not the kernel's state. It exists
to report the rule; nothing may branch on it (vllm/envs.py "NOT A PYTHON
READ SITE" still holds: this module decides nothing).
* ``cpp-cache:`` -- the kernel's switches as the C++ statics cached them, from
``torch.ops._C.persistent_topk_toggle_state`` and
``torch.ops._moe_C.marlin_moe_full_k_state`` (query ops that only LOAD a
mirror the static initialisers publish; they never fill a cache). ``n-a``
with a reason on a build without those ops -- the extensions this release
was first cut against (earlier v2.2 builds, not published) have none.
* ``python:`` -- ``qsa_indexer._QSA_SORT``, the module constant the QSA sort
actually branches on (looked up in ``sys.modules``, never imported here;
``n-a`` if the module was never loaded in this process).
* ``device:`` -- counter slots [8]/[9], which the counter op fills from the
C++ ``exact_medium_enabled()`` / ``det_large_enabled()`` statics
(topk.cu persistent_topk_exact_counters).
WHEN. Once per worker, after ``compile_or_warm_up_model`` has run the MRV2
kernel warmup and cudagraph capture (``arm()``), i.e. after this process's
first persistent_topk and Marlin MoE launches -- NOT at the counter reader's
first read: an integration boot logged its first counter line BEFORE
the first tie-policy line in the engine log, because the
first ``maybe_log`` runs at the head of the warmup's first execute step, before
that step's forward. When the cpp-cache ops exist and a static is still
uninitialised at ``arm()``, the line is deferred to the head of the first
execute step where every static is filled, and forced out at shutdown
otherwise (with ``uninit`` values, which the smoke FAILs). It does not depend on
``MERLIN_TOPK_EXACT_COUNTERS_LOG``: it fires with the counter log off.
WHAT THIS DOES NOT PROVE. Without the cpp-cache ops the C++ state is inferred,
not read: for FULL_K, TIE_DET, TIE_RECENT and EXACT_DECODE the line is the
environment at emission time run through the C++ parser's rules, and the only
C++-derived evidence is ``device:`` (EXACT_MEDIUM and DET_LARGE) plus the
untagged stderr policy line. The residual gap is an environment mutated
between the kernel's first ``getenv`` and this read. Nothing in this tree does
that (by grep of csrc/): no ``setenv``/``putenv``/``unsetenv`` under csrc/, and
the only runtime ``os.environ.update`` calls in vllm/ are the RunAI streamer's
fixed ``RUNAI_STREAMER_*`` keys (model_executor/model_loader/
runai_streamer_loader.py:73) and the Ray executor's ``override_env_vars``
(v1/executor/ray_utils.py:180), which the multiproc executor never calls. With the
cpp-cache ops that gap closes for every C++ name. Even then the line proves the
CONFIGURATION each rank ran, not that a switch changed any output: that is the
job of an output-equality run (identical sends -> one distinct output).
"""
from __future__ import annotations
import os
import sys
from collections.abc import Callable
from vllm.logger import init_logger
logger = init_logger(__name__)
# (environment name, effective key, value when unset or empty) -- the C++
# readers, each `static const bool enabled = [] { ... }();` once per process:
# MERLIN_FULL_K csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu:57-63
# MERLIN_TIE_DET csrc/libtorch_stable/topk.cu:60-66
# MERLIN_TIE_RECENT csrc/libtorch_stable/topk.cu:52-58
# MERLIN_TOPK_EXACT_MEDIUM csrc/libtorch_stable/topk.cu:68-74 (unset -> ON)
# MERLIN_TOPK_EXACT_DECODE csrc/libtorch_stable/topk.cu:103-109
# MERLIN_TOPK_DET_LARGE csrc/libtorch_stable/topk.cu:86-92 (unset -> ON)
CXX_SWITCHES: tuple[tuple[str, str, bool], ...] = (
("MERLIN_FULL_K", "full_k", False),
("MERLIN_TIE_DET", "tie_det", False),
("MERLIN_TIE_RECENT", "tie_recent", False),
("MERLIN_TOPK_EXACT_MEDIUM", "exact_medium", True),
("MERLIN_TOPK_EXACT_DECODE", "exact_decode", False),
("MERLIN_TOPK_DET_LARGE", "det_large", True),
)
QSA_SORT_NAME = "MERLIN_QSA_SORT"
RAW_ORDER = (
"MERLIN_FULL_K",
"MERLIN_QSA_SORT",
"MERLIN_TIE_DET",
"MERLIN_TIE_RECENT",
"MERLIN_TOPK_EXACT_MEDIUM",
"MERLIN_TOPK_EXACT_DECODE",
"MERLIN_TOPK_DET_LARGE",
)
EFFECTIVE_ORDER = (
"full_k",
"qsa_sort",
"tie_det",
"tie_recent",
"tie_recent_effective",
"exact_medium",
"exact_decode",
"det_large",
)
CPP_ORDER = tuple(k for k in EFFECTIVE_ORDER if k != "qsa_sort")
_QSA_MODULE = "vllm.models.qwen4_exp.nvidia.ops.qsa_indexer"
_C_SPACE = b" \t\n\v\f\r" # isspace() in the "C" locale
_LONG_MIN, _LONG_MAX = -(2**63), 2**63 - 1 # LP64 long, the reference host's glibc
# --------------------------------------------------------------- parsers ----
def c_atoi(raw: bytes) -> int:
"""glibc ``atoi``: ``(int) strtol(raw, NULL, 10)`` (glibc stdlib/atoi.c).
Leading C-locale whitespace, one optional sign, then decimal digits up to
the first non-digit; no digits -> 0. ``strtol`` clamps to the ``long``
range and the ``(int)`` conversion keeps the low 32 bits (GCC's documented
modulo behaviour). So ``"true"`` -> 0, ``"1abc"`` -> 1, ``"0x1"`` -> 0,
``" +1"`` -> 1, ``"+ 1"`` -> 0 and ``"4294967296"`` -> 0.
"""
i, n = 0, len(raw)
while i < n and raw[i] in _C_SPACE:
i += 1
negative = False
if i < n and raw[i] in b"+-":
negative = raw[i] == ord("-")
i += 1
value, digits = 0, 0
while i < n and 0x30 <= raw[i] <= 0x39:
value = value * 10 + (raw[i] - 0x30)
digits += 1
i += 1
if digits == 0:
return 0
if negative:
value = -value
value = max(_LONG_MIN, min(_LONG_MAX, value))
value &= 0xFFFFFFFF
return value - (1 << 32) if value >= (1 << 31) else value
def cxx_switch(raw: bytes | None, unset_value: bool) -> bool:
"""The C++ getter body, both shapes:
default OFF ``(raw != nullptr && raw[0] != '\\0') ? (atoi(raw) != 0) : false``
default ON ``(raw == nullptr || raw[0] == '\\0') ? true : (atoi(raw) != 0)``
An environment value cannot contain NUL, so "empty" is exactly
``raw[0] == '\\0'``."""
if raw is None or raw == b"":
return unset_value
return c_atoi(raw) != 0
def qsa_sort_value(raw: str | None) -> str:
"""vllm/envs.py:2372 ``bool(int(os.getenv("MERLIN_QSA_SORT", "0")))``,
read once at import by qsa_indexer.py:51 (``_QSA_SORT``). ``int()`` raises
on ``"true"`` and on ``""`` -- that engine would not have booted -- which is
reported as ``invalid`` rather than guessed. Python's own ``int`` is used,
so whitespace, ``+``/``-``, ``1_0`` and non-ASCII digits parse exactly as
the reader parses them."""
try:
return "1" if bool(int("0" if raw is None else raw)) else "0"
except ValueError:
return "invalid"
def effective_tie_recent(det_ties: bool, tie_recent: bool) -> bool:
"""csrc/libtorch_stable/persistent_topk.cuh:65-68: ``det_ties ? tie_recent
: true`` -- with TIE_DET off the restart paths keep most-recent-wins."""
return tie_recent if det_ties else True
# --------------------------------------------------------------- sources ----
def _libc_getenv() -> tuple[Callable[[str], bytes | None], str]:
"""libc ``getenv``: the ``environ`` the C++ ``std::getenv`` calls read.
Falls back to ``os.environb`` / ``os.environ`` -- and says so in the line's
``source=`` -- where there is no C library to ask (an offline run off Linux).
"""
try:
import ctypes
getenv = ctypes.CDLL(None).getenv
getenv.restype = ctypes.c_char_p
getenv.argtypes = [ctypes.c_char_p]
def read(name: str) -> bytes | None:
return getenv(name.encode("ascii"))
return read, "libc-getenv"
except (OSError, AttributeError, TypeError):
pass
environb = getattr(os, "environb", None)
if environb is not None:
return (lambda name: environb.get(name.encode("ascii"))), "os-environ"
def read_environ(name: str) -> bytes | None:
value = os.environ.get(name)
return None if value is None else value.encode("utf-8", "surrogateescape")
return read_environ, "os-environ"
def _printable(b: int) -> bool:
return 0x21 <= b <= 0x7E and b not in b'"\\'
def encode_raw(raw: bytes | None) -> str:
"""``unset``, a plain token, or a quoted, hex-escaped string -- never a
space, so the line splits on whitespace. The literal value ``unset`` is
quoted so it cannot pass for an absent variable."""
if raw is None:
return "unset"
if raw and raw != b"unset" and all(_printable(b) for b in raw):
return raw.decode("ascii")
return '"' + "".join(chr(b) if _printable(b) else f"\\x{b:02x}" for b in raw) + '"'
def environ_view(
cxx_getenv: Callable[[str], bytes | None],
py_getenv: Callable[[str], str | None],
) -> tuple[dict[str, str], dict[str, str]]:
"""(raw value by name, effective map) from one read of the environment."""
raw: dict[str, str] = {}
flags: dict[str, bool] = {}
for name, key, unset_value in CXX_SWITCHES:
value = cxx_getenv(name)
raw[name] = encode_raw(value)
flags[key] = cxx_switch(value, unset_value)
qsa_raw = py_getenv(QSA_SORT_NAME)
raw[QSA_SORT_NAME] = encode_raw(
None if qsa_raw is None else qsa_raw.encode("utf-8", "surrogateescape")
)
eff = {key: "1" if flags[key] else "0" for key in flags}
eff["qsa_sort"] = qsa_sort_value(qsa_raw)
eff["tie_recent_effective"] = (
"1" if effective_tie_recent(flags["tie_det"], flags["tie_recent"]) else "0"
)
return raw, {key: eff[key] for key in EFFECTIVE_ORDER}
_TOPK_STATE_FIELDS = (
"tie_det",
"tie_recent",
"tie_recent_effective",
"exact_medium",
"exact_decode",
"det_large",
)
def decode_topk_state(state: int) -> dict[str, str] | str:
"""Unpack ``persistent_topk_toggle_state()``: 2 bits per field (0 off,
1 on, 2 not initialised), field count at bits 48-55 (6), layout version at
bits 56-62 (1). Any other layout is refused with a reason string."""
version, count = (state >> 56) & 0x7F, (state >> 48) & 0xFF
if version != 1 or count != len(_TOPK_STATE_FIELDS):
return f"unknown-layout-v{version}-{count}-fields"
names = {0: "0", 1: "1", 2: "uninit"}
return {
key: names.get((state >> (2 * i)) & 3, "invalid")
for i, key in enumerate(_TOPK_STATE_FIELDS)
}
def decode_full_k_state(state: int) -> str:
"""``marlin_moe_full_k_state()``: -1 not initialised, 0 off, 1 on."""
return {-1: "uninit", 0: "0", 1: "1"}.get(int(state), "invalid")
def _op(namespace: str, name: str):
try:
import torch
return getattr(getattr(torch.ops, namespace), name, None)
except (AttributeError, RuntimeError, ImportError):
return None
def cpp_cache() -> dict[str, str] | str:
"""The C++ cached switches, or a reason string when the build cannot say."""
topk_op = _op("_C", "persistent_topk_toggle_state")
full_k_op = _op("_moe_C", "marlin_moe_full_k_state")
missing = [
name
for name, op in (
("_C.persistent_topk_toggle_state", topk_op),
("_moe_C.marlin_moe_full_k_state", full_k_op),
)
if op is None
]
if missing:
return "op-absent:" + ",".join(missing)
decoded = decode_topk_state(int(topk_op()))
if isinstance(decoded, str):
return decoded
out = dict(decoded, full_k=decode_full_k_state(int(full_k_op())))
return {key: out[key] for key in CPP_ORDER}
def qsa_sort_live() -> str:
module = sys.modules.get(_QSA_MODULE)
if module is None or not hasattr(module, "_QSA_SORT"):
return "n-a"
return "1" if bool(module._QSA_SORT) else "0"
def device_switches() -> dict[str, str]:
"""Counter slots [8]/[9] (switch / switch_large). One synchronizing read
(topk_counters.read refuses inside a capture), so it is only called from
arm() and the head of an execute step."""
try:
from vllm.models.qwen4_exp.nvidia.ops import topk_counters
counters = topk_counters.read()
except Exception:
counters = {}
return {
key: str(counters[key]) if key in counters else "n-a"
for key in ("switch", "switch_large")
}
# ------------------------------------------------------------------ line ----
def toggles_line(
tag: str,
pid: int,
raw: dict[str, str],
effective: dict[str, str],
env_source: str,
cpp: dict[str, str] | str,
qsa_live: str,
device: dict[str, str],
) -> str:
parts = [f"e2-toggles[{tag}]: pid={pid}"]
parts += [f"{name}={raw[name]}" for name in RAW_ORDER]
parts.append("effective:")
parts += [f"{key}={effective[key]}" for key in EFFECTIVE_ORDER]
parts.append(f"source={env_source}")
parts.append("cpp-cache:")
if isinstance(cpp, str):
parts += ["n-a", f"reason={cpp}"]
else:
parts += [f"{key}={cpp[key]}" for key in CPP_ORDER]
parts.append("source=cpp-cache")
parts += ["python:", f"qsa_sort={qsa_live}", "source=qsa_indexer._QSA_SORT"]
parts.append("device:")
parts += [f"{key}={device[key]}" for key in ("switch", "switch_large")]
parts.append("source=counter-slots-8-9")
return " ".join(parts)
def _rank_tag() -> str:
from vllm.distributed.parallel_state import get_pp_group, get_tp_group
return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}"
# ----------------------------------------------------------------- state ----
_armed = False
_emitted = False
_deferral_noted = False
def _reset_for_tests() -> None:
global _armed, _emitted, _deferral_noted
_armed = _emitted = _deferral_noted = False
def arm(**kwargs) -> str:
"""Once per worker, at the end of compile_or_warm_up_model (warmup and
capture done): emit now, unless a C++ static is still uninitialised."""
global _armed
_armed = True
return maybe_emit(**kwargs)
def maybe_emit(
force: bool = False,
*,
tag_fn: Callable[[], str] = _rank_tag,
getenv_fn: Callable[[], tuple[Callable[[str], bytes | None], str]] = _libc_getenv,
cpp_fn: Callable[[], dict[str, str] | str] = cpp_cache,
device_fn: Callable[[], dict[str, str]] = device_switches,
qsa_fn: Callable[[], str] = qsa_sort_live,
) -> str:
"""Emit the line at most once per process. One bool check when already
emitted or not yet armed, so it can sit at the head of every execute step.
Never raises. Returns the line when this call emitted it, else ""."""
global _emitted, _deferral_noted
if _emitted or not _armed:
return ""
tag = "PP?_TP?"
try:
tag = tag_fn()
cpp = cpp_fn()
pending = (
sorted(k for k, v in cpp.items() if v == "uninit")
if isinstance(cpp, dict)
else []
)
if pending and not force:
if not _deferral_noted:
_deferral_noted = True
# Deliberately not 'e2-toggles[': the smoke counts those.
logger.info(
"e2-toggles deferred on %s: C++ cache not initialised for %s",
tag,
",".join(pending),
)
return ""
cxx_getenv, env_source = getenv_fn()
raw, effective = environ_view(cxx_getenv, os.environ.get)
line = toggles_line(
tag, os.getpid(), raw, effective, env_source, cpp, qsa_fn(), device_fn()
)
_emitted = True
logger.info("%s", line)
return line
except Exception as error: # log-only: must never break a boot or a step
_emitted = True
logger.warning(
"e2-toggles[%s]: unavailable (%s: %s)", tag, type(error).__name__, error
)
return ""
+6 -1
View File
@@ -85,7 +85,7 @@ from vllm.v1.watermarking.spec_decode import (
create_speculative_target_watermarker,
speculative_target_watermark_key,
)
from vllm.v1.worker import guard_telemetry
from vllm.v1.worker import e2_toggles, guard_telemetry
from vllm.v1.worker.block_table import get_block_table_width
from vllm.v1.worker.cp_utils import check_attention_cp_compatibility
from vllm.v1.worker.gpu import pcp_manager as pcp
@@ -1863,6 +1863,9 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# never fire on them. Outside any capture: this is the engine
# thread between steps (read() re-checks).
topk_counters.maybe_log()
# v2.2: the e2-toggles line if arm() had to defer it (a C++ static
# still uninitialised). One bool check once it has been emitted.
e2_toggles.maybe_emit()
# Update the request states.
self.update_pp_decode_requests()
self.finish_requests(scheduler_output)
@@ -2580,6 +2583,8 @@ class GPUModelRunner(LoRAModelRunnerMixin):
# The last window of a run is the one a cadence log always
# misses; emit it before the device state goes away.
topk_counters.maybe_log(force=True)
# v2.2: a still-deferred e2-toggles line goes out now, 'uninit' and all.
e2_toggles.maybe_emit(force=True)
if self._ple_offload_connector is not None:
self._ple_offload_connector.close()
self._ple_offload_connector = None
+13
View File
@@ -740,6 +740,12 @@ class Worker(WorkerBase):
model_runner = cast("GPUModelRunnerV2", self.model_runner)
model_runner._setup_ple_offload(self.parallel_config._ple_offload_ipc_path)
# Log-only: this rank's PLE / capture identity for target and drafter,
# read after the offload placeholders are armed (v1/worker/e2_identity.py).
from vllm.v1.worker.e2_identity import log_launch_identity
log_launch_identity(self.model_runner, self.vllm_config)
if has_ec_transfer():
get_ec_transfer().start_worker_services()
@@ -1093,6 +1099,13 @@ class Worker(WorkerBase):
with self._get_cudagraph_capture_context():
cuda_graph_memory_bytes = self.model_runner.capture_model()
# Log-only: this rank's determinism toggles, once, now that the warmup
# and capture above have launched persistent_topk and Marlin MoE, so
# their C++ statics are filled (v1/worker/e2_toggles.py). Never raises.
from vllm.v1.worker import e2_toggles
e2_toggles.arm()
# Compare actual vs estimated CUDA graph memory (if we did profiling)
if (
hasattr(self, "cudagraph_memory_estimate")