[Worker] E2-identity and rank-tagged e2-toggles startup lines
Each worker logs its launch identity and, after warm-up and capture, one e2-toggles line with the effective toggle state read back from the C++ extensions through two new tensor-free query ops. Both extensions must be rebuilt. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -33,6 +33,7 @@
|
||||
#include <torch/headeronly/util/Exception.h>
|
||||
|
||||
#include "libtorch_stable/torch_utils.h"
|
||||
#include <atomic>
|
||||
#include <cstdlib>
|
||||
|
||||
namespace {
|
||||
@@ -53,14 +54,31 @@ namespace {
|
||||
// This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw
|
||||
// passthrough entry for validate_environ and the compile-factor hash only;
|
||||
// nothing in Python may parse this name a second time.
|
||||
//
|
||||
// The initialiser also
|
||||
// publishes the value it cached into g_cached_full_k (-1 = not initialised
|
||||
// yet; nothing else writes it), and marlin_moe_full_k_state() below only
|
||||
// LOADS it -- asking never fills the cache. Host code only.
|
||||
// -------------------------------------------------------------------------
|
||||
std::atomic<int> g_cached_full_k{-1};
|
||||
|
||||
bool merlin_full_k() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_FULL_K");
|
||||
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||
const bool value =
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||
g_cached_full_k.store(value ? 1 : 0, std::memory_order_release);
|
||||
return value;
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// MERLIN_FULL_K as this process cached it: -1 = not initialised yet (no Marlin
|
||||
// MoE launch has happened in this process), 0 = off, 1 = on. Tensor-free, no
|
||||
// CUDA call.
|
||||
int64_t marlin_moe_full_k_state() {
|
||||
return static_cast<int64_t>(g_cached_full_k.load(std::memory_order_acquire));
|
||||
}
|
||||
} // namespace
|
||||
|
||||
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
|
||||
@@ -736,3 +754,14 @@ torch::stable::Tensor moe_wna16_marlin_gemm(
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) {
|
||||
m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm));
|
||||
}
|
||||
|
||||
// v2.2 -- defined and registered in this TU so the query exists exactly when
|
||||
// the static it reads does. Tensor-free, so CompositeExplicitAutograd (a CUDA
|
||||
// registration could never be dispatched to without a tensor argument).
|
||||
STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) {
|
||||
m.def("marlin_moe_full_k_state() -> int");
|
||||
}
|
||||
|
||||
STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) {
|
||||
m.impl("marlin_moe_full_k_state", TORCH_BOX(&marlin_moe_full_k_state));
|
||||
}
|
||||
|
||||
@@ -489,6 +489,10 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||
// reader never has to guess it from an environment variable.
|
||||
int64_t persistent_topk_exact_counters_slots();
|
||||
|
||||
// The persistent_topk switches as this
|
||||
// process cached them, packed; see topk.cu for the layout.
|
||||
int64_t persistent_topk_toggle_state();
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
void cooperative_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
|
||||
@@ -44,6 +44,29 @@ namespace {
|
||||
// This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw
|
||||
// passthrough for validate_environ and the compile-factor hash only.
|
||||
// -------------------------------------------------------------------------
|
||||
// -------------------------------------------------------------------------
|
||||
// A READ-ONLY MIRROR of the
|
||||
// switches as this process CACHED them, for persistent_topk_toggle_state().
|
||||
//
|
||||
// Each switch below is a function-local static filled once by its lambda. The
|
||||
// lambda now also stores the value it cached into the matching atomic here
|
||||
// (publish_cached), and nothing else ever writes these atomics, so a mirror
|
||||
// either says "not initialised yet" (-1) or holds exactly the cached value.
|
||||
// The query only LOADS them: it never calls a getter, so asking cannot fill a
|
||||
// cache from an environment that differs from the one the kernel saw. The
|
||||
// parse expressions are unchanged; this is host code only.
|
||||
// -------------------------------------------------------------------------
|
||||
std::atomic<int> g_cached_tie_recent{-1};
|
||||
std::atomic<int> g_cached_tie_det{-1};
|
||||
std::atomic<int> g_cached_exact_medium{-1};
|
||||
std::atomic<int> g_cached_det_large{-1};
|
||||
std::atomic<int> g_cached_exact_decode{-1};
|
||||
|
||||
bool publish_cached(std::atomic<int>& mirror, bool value) {
|
||||
mirror.store(value ? 1 : 0, std::memory_order_release);
|
||||
return value;
|
||||
}
|
||||
|
||||
// MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above.
|
||||
// Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode,
|
||||
// most-recent-wins is +1.5 %. Both deterministic. Also the direction
|
||||
@@ -53,7 +76,9 @@ namespace {
|
||||
bool tie_recent_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TIE_RECENT");
|
||||
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||
return publish_cached(
|
||||
g_cached_tie_recent,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
@@ -61,7 +86,9 @@ bool tie_recent_enabled() {
|
||||
bool det_ties_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TIE_DET");
|
||||
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||
return publish_cached(
|
||||
g_cached_tie_det,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
@@ -69,7 +96,9 @@ bool det_ties_enabled() {
|
||||
bool exact_medium_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
|
||||
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
|
||||
return publish_cached(
|
||||
g_cached_exact_medium,
|
||||
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
@@ -87,7 +116,9 @@ bool exact_medium_enabled() {
|
||||
bool det_large_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_DET_LARGE");
|
||||
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
|
||||
return publish_cached(
|
||||
g_cached_det_large,
|
||||
(raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0));
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
@@ -104,7 +135,9 @@ bool det_large_enabled() {
|
||||
bool exact_decode_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE");
|
||||
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
|
||||
return publish_cached(
|
||||
g_cached_exact_decode,
|
||||
(raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
@@ -887,3 +920,49 @@ int64_t persistent_topk_exact_counters_slots() {
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
// The persistent_topk
|
||||
// switches AS THIS PROCESS CACHED THEM -- the values every launch of this
|
||||
// process uses -- read from the mirrors above. Tensor-free, no CUDA call, no
|
||||
// allocation, never initialises a cache (see publish_cached), so it is safe
|
||||
// from anywhere and asking changes nothing.
|
||||
//
|
||||
// Packed into one int64 so the op needs no tensor and no dispatch key.
|
||||
// 2 bits per field, field i at bits [2i, 2i+1]: 0 = off, 1 = on,
|
||||
// 2 = the static has not been initialised in this process yet.
|
||||
// field 0 MERLIN_TIE_DET (det_ties_enabled)
|
||||
// field 1 MERLIN_TIE_RECENT as parsed (tie_recent_enabled)
|
||||
// field 2 effective direction, 1 = most-recent (effective_tie_recent)
|
||||
// field 3 MERLIN_TOPK_EXACT_MEDIUM (exact_medium_enabled)
|
||||
// field 4 MERLIN_TOPK_EXACT_DECODE (exact_decode_enabled)
|
||||
// field 5 MERLIN_TOPK_DET_LARGE (det_large_enabled)
|
||||
// bits [48, 55] = field count (6), bits [56, 62] = layout version (1). A
|
||||
// reader must check both and refuse any other layout rather than guess.
|
||||
int64_t persistent_topk_toggle_state() {
|
||||
#ifndef USE_ROCM
|
||||
const auto field = [](int cached) -> int64_t {
|
||||
return cached < 0 ? 2 : (cached != 0 ? 1 : 0);
|
||||
};
|
||||
const int det = g_cached_tie_det.load(std::memory_order_acquire);
|
||||
const int recent = g_cached_tie_recent.load(std::memory_order_acquire);
|
||||
// The kernel's own rule, from the kernel's own header, on the cached values.
|
||||
const int effective =
|
||||
(det < 0 || recent < 0)
|
||||
? -1
|
||||
: (vllm::persistent::effective_tie_recent(det != 0, recent != 0)
|
||||
? 1
|
||||
: 0);
|
||||
int64_t state = 0;
|
||||
state |= field(det) << 0;
|
||||
state |= field(recent) << 2;
|
||||
state |= field(effective) << 4;
|
||||
state |= field(g_cached_exact_medium.load(std::memory_order_acquire)) << 6;
|
||||
state |= field(g_cached_exact_decode.load(std::memory_order_acquire)) << 8;
|
||||
state |= field(g_cached_det_large.load(std::memory_order_acquire)) << 10;
|
||||
state |= int64_t{6} << 48;
|
||||
state |= int64_t{1} << 56;
|
||||
return state;
|
||||
#else
|
||||
return 0; // layout version 0: no persistent_topk on ROCm
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -590,6 +590,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
ops.def(
|
||||
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
|
||||
ops.def("persistent_topk_exact_counters_slots() -> int");
|
||||
// v2.2 -- the switches as this process cached them (topk.cu for the layout).
|
||||
ops.def("persistent_topk_toggle_state() -> int");
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.def(
|
||||
@@ -947,6 +949,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) {
|
||||
// NotImplementedError "no fallback function is registered").
|
||||
ops.impl("persistent_topk_exact_counters_slots",
|
||||
TORCH_BOX(&persistent_topk_exact_counters_slots));
|
||||
// Tensor-free for the same reason as the slot count above.
|
||||
ops.impl("persistent_topk_toggle_state",
|
||||
TORCH_BOX(&persistent_topk_toggle_state));
|
||||
}
|
||||
|
||||
// Cache ops
|
||||
|
||||
Reference in New Issue
Block a user