[Worker] E2-identity and rank-tagged e2-toggles startup lines

Each worker logs its launch identity and, after warm-up and capture, one
e2-toggles line with the effective toggle state read back from the C++
extensions through two new tensor-free query ops. Both extensions must be
rebuilt.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:56:00 +07:00
co-authored by Hinata
parent bef464a69d
commit 281491646d
8 changed files with 734 additions and 7 deletions
@@ -33,6 +33,7 @@
#include <torch/headeronly/util/Exception.h>
#include "libtorch_stable/torch_utils.h"
#include <atomic>
#include <cstdlib>
namespace {
@@ -53,14 +54,31 @@ namespace {
// This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw
// passthrough entry for validate_environ and the compile-factor hash only;
// nothing in Python may parse this name a second time.
//
// The initialiser also
// publishes the value it cached into g_cached_full_k (-1 = not initialised
// yet; nothing else writes it), and marlin_moe_full_k_state() below only
// LOADS it -- asking never fills the cache. Host code only.
// -------------------------------------------------------------------------
std::atomic<int> g_cached_full_k{-1};
bool merlin_full_k() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_FULL_K");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
const bool value =
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
g_cached_full_k.store(value ? 1 : 0, std::memory_order_release);
return value;
}();
return enabled;
}
// MERLIN_FULL_K as this process cached it: -1 = not initialised yet (no Marlin
// MoE launch has happened in this process), 0 = off, 1 = on. Tensor-free, no
// CUDA call.
int64_t marlin_moe_full_k_state() {
return static_cast<int64_t>(g_cached_full_k.load(std::memory_order_acquire));
}
} // namespace
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
@@ -736,3 +754,14 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
}
// v2.2 -- defined and registered in this TU so the query exists exactly when
// the static it reads does. Tensor-free, so CompositeExplicitAutograd (a CUDA
// registration could never be dispatched to without a tensor argument).
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
m.def("marlin_moe_full_k_state() -> int");
}
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
m.impl("marlin_moe_full_k_state", TORCH_BOX(&marlin_moe_full_k_state));
}
+4
View File
@@ -489,6 +489,10 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
// reader never has to guess it from an environment variable.
int64_t persistent_topk_exact_counters_slots();
// The persistent_topk switches as this
// process cached them, packed; see topk.cu for the layout.
int64_t persistent_topk_toggle_state();
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
void cooperative_topk(const torch::stable::Tensor& logits,
const torch::stable::Tensor& lengths,
+84 -5
View File
@@ -44,6 +44,29 @@ namespace {
// This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw
// passthrough for validate_environ and the compile-factor hash only.
// -------------------------------------------------------------------------
// -------------------------------------------------------------------------
// A READ-ONLY MIRROR of the
// switches as this process CACHED them, for persistent_topk_toggle_state().
//
// Each switch below is a function-local static filled once by its lambda. The
// lambda now also stores the value it cached into the matching atomic here
// (publish_cached), and nothing else ever writes these atomics, so a mirror
// either says "not initialised yet" (-1) or holds exactly the cached value.
// The query only LOADS them: it never calls a getter, so asking cannot fill a
// cache from an environment that differs from the one the kernel saw. The
// parse expressions are unchanged; this is host code only.
// -------------------------------------------------------------------------
std::atomic<int> g_cached_tie_recent{-1};
std::atomic<int> g_cached_tie_det{-1};
std::atomic<int> g_cached_exact_medium{-1};
std::atomic<int> g_cached_det_large{-1};
std::atomic<int> g_cached_exact_decode{-1};
bool publish_cached(std::atomic<int>& mirror, bool value) {
mirror.store(value ? 1 : 0, std::memory_order_release);
return value;
}
// MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above.
// Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode,
// most-recent-wins is +1.5 %. Both deterministic. Also the direction
@@ -53,7 +76,9 @@ namespace {
bool tie_recent_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TIE_RECENT");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
return publish_cached(
g_cached_tie_recent,
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
}();
return enabled;
}
@@ -61,7 +86,9 @@ bool tie_recent_enabled() {
bool det_ties_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TIE_DET");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
return publish_cached(
g_cached_tie_det,
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
}();
return enabled;
}
@@ -69,7 +96,9 @@ bool det_ties_enabled() {
bool exact_medium_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
return publish_cached(
g_cached_exact_medium,
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
}();
return enabled;
}
@@ -87,7 +116,9 @@ bool exact_medium_enabled() {
bool det_large_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_DET_LARGE");
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
return publish_cached(
g_cached_det_large,
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
}();
return enabled;
}
@@ -104,7 +135,9 @@ bool det_large_enabled() {
bool exact_decode_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
return publish_cached(
g_cached_exact_decode,
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
}();
return enabled;
}
@@ -887,3 +920,49 @@ int64_t persistent_topk_exact_counters_slots() {
return 0;
#endif
}
// The persistent_topk
// switches AS THIS PROCESS CACHED THEM -- the values every launch of this
// process uses -- read from the mirrors above. Tensor-free, no CUDA call, no
// allocation, never initialises a cache (see publish_cached), so it is safe
// from anywhere and asking changes nothing.
//
// Packed into one int64 so the op needs no tensor and no dispatch key.
// 2 bits per field, field i at bits [2i, 2i+1]: 0 = off, 1 = on,
// 2 = the static has not been initialised in this process yet.
// field 0 MERLIN_TIE_DET (det_ties_enabled)
// field 1 MERLIN_TIE_RECENT as parsed (tie_recent_enabled)
// field 2 effective direction, 1 = most-recent (effective_tie_recent)
// field 3 MERLIN_TOPK_EXACT_MEDIUM (exact_medium_enabled)
// field 4 MERLIN_TOPK_EXACT_DECODE (exact_decode_enabled)
// field 5 MERLIN_TOPK_DET_LARGE (det_large_enabled)
// bits [48, 55] = field count (6), bits [56, 62] = layout version (1). A
// reader must check both and refuse any other layout rather than guess.
int64_t persistent_topk_toggle_state() {
#ifndef USE_ROCM
const auto field = [](int cached) -> int64_t {
return cached < 0 ? 2 : (cached != 0 ? 1 : 0);
};
const int det = g_cached_tie_det.load(std::memory_order_acquire);
const int recent = g_cached_tie_recent.load(std::memory_order_acquire);
// The kernel's own rule, from the kernel's own header, on the cached values.
const int effective =
(det < 0 || recent < 0)
? -1
: (vllm::persistent::effective_tie_recent(det != 0, recent != 0)
? 1
: 0);
int64_t state = 0;
state |= field(det) << 0;
state |= field(recent) << 2;
state |= field(effective) << 4;
state |= field(g_cached_exact_medium.load(std::memory_order_acquire)) << 6;
state |= field(g_cached_exact_decode.load(std::memory_order_acquire)) << 8;
state |= field(g_cached_det_large.load(std::memory_order_acquire)) << 10;
state |= int64_t{6} << 48;
state |= int64_t{1} << 56;
return state;
#else
return 0; // layout version 0: no persistent_topk on ROCm
#endif
}
+5
View File
@@ -590,6 +590,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
ops.def(
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
ops.def("persistent_topk_exact_counters_slots() -> int");
// v2.2 -- the switches as this process cached them (topk.cu for the layout).
ops.def("persistent_topk_toggle_state() -> int");
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
ops.def(
@@ -947,6 +949,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
// NotImplementedError "no fallback function is registered").
ops.impl("persistent_topk_exact_counters_slots",
TORCH_BOX(&persistent_topk_exact_counters_slots));
// Tensor-free for the same reason as the slot count above.
ops.impl("persistent_topk_toggle_state",
TORCH_BOX(&persistent_topk_toggle_state));
}
// Cache ops