From 281491646da54580d3b4c702723987dc2988d0db Mon Sep 17 00:00:00 2001 From: MOVZX Date: Wed, 23 Sep 2026 08:41:02 -0400 Subject: [PATCH] [Worker] E2-identity and rank-tagged e2-toggles startup lines Each worker logs its launch identity and, after warm-up and capture, one e2-toggles line with the effective toggle state read back from the C++ extensions through two new tensor-free query ops. Both extensions must be rebuilt. Co-Authored-By: Hinata Signed-off-by: MOVZX --- .../moe/marlin_moe_wna16/ops.cu | 31 +- csrc/libtorch_stable/ops.h | 4 + csrc/libtorch_stable/topk.cu | 89 +++- csrc/libtorch_stable/torch_bindings.cpp | 5 + vllm/v1/worker/e2_identity.py | 148 ++++++ vllm/v1/worker/e2_toggles.py | 444 ++++++++++++++++++ vllm/v1/worker/gpu/model_runner.py | 7 +- vllm/v1/worker/gpu_worker.py | 13 + 8 files changed, 734 insertions(+), 7 deletions(-) create mode 100644 vllm/v1/worker/e2_identity.py create mode 100644 vllm/v1/worker/e2_toggles.py diff --git a/csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu b/csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu index 2c6ad7666a..0a6f5de67d 100644 --- a/csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu +++ b/csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu @@ -33,6 +33,7 @@ #include #include "libtorch_stable/torch_utils.h" +#include #include namespace { @@ -53,14 +54,31 @@ namespace { // This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw // passthrough entry for validate_environ and the compile-factor hash only; // nothing in Python may parse this name a second time. +// +// The initialiser also +// publishes the value it cached into g_cached_full_k (-1 = not initialised +// yet; nothing else writes it), and marlin_moe_full_k_state() below only +// LOADS it -- asking never fills the cache. Host code only. // ------------------------------------------------------------------------- +std::atomic g_cached_full_k{-1}; + bool merlin_full_k() { static const bool enabled = [] { const char* raw = std::getenv("MERLIN_FULL_K"); - return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false; + const bool value = + (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false; + g_cached_full_k.store(value ? 1 : 0, std::memory_order_release); + return value; }(); return enabled; } + +// MERLIN_FULL_K as this process cached it: -1 = not initialised yet (no Marlin +// MoE launch has happened in this process), 0 = off, 1 = on. Tensor-free, no +// CUDA call. +int64_t marlin_moe_full_k_state() { + return static_cast(g_cached_full_k.load(std::memory_order_acquire)); +} } // namespace #define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \ @@ -736,3 +754,14 @@ torch::stable::Tensor moe_wna16_marlin_gemm( STABLE_TORCH_LIBRARY_IMPL(_moe_C, CUDA, m) { m.impl("moe_wna16_marlin_gemm", TORCH_BOX(&moe_wna16_marlin_gemm)); } + +// v2.2 -- defined and registered in this TU so the query exists exactly when +// the static it reads does. Tensor-free, so CompositeExplicitAutograd (a CUDA +// registration could never be dispatched to without a tensor argument). +STABLE_TORCH_LIBRARY_FRAGMENT(_moe_C, m) { + m.def("marlin_moe_full_k_state() -> int"); +} + +STABLE_TORCH_LIBRARY_IMPL(_moe_C, CompositeExplicitAutograd, m) { + m.impl("marlin_moe_full_k_state", TORCH_BOX(&marlin_moe_full_k_state)); +} diff --git a/csrc/libtorch_stable/ops.h b/csrc/libtorch_stable/ops.h index ca97335d7b..219e67885a 100644 --- a/csrc/libtorch_stable/ops.h +++ b/csrc/libtorch_stable/ops.h @@ -489,6 +489,10 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset, // reader never has to guess it from an environment variable. int64_t persistent_topk_exact_counters_slots(); +// The persistent_topk switches as this +// process cached them, packed; see topk.cu for the layout. +int64_t persistent_topk_toggle_state(); + #ifdef VLLM_ENABLE_COOPERATIVE_TOPK void cooperative_topk(const torch::stable::Tensor& logits, const torch::stable::Tensor& lengths, diff --git a/csrc/libtorch_stable/topk.cu b/csrc/libtorch_stable/topk.cu index c21524ee96..26d5dfbcb3 100644 --- a/csrc/libtorch_stable/topk.cu +++ b/csrc/libtorch_stable/topk.cu @@ -44,6 +44,29 @@ namespace { // This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw // passthrough for validate_environ and the compile-factor hash only. // ------------------------------------------------------------------------- +// ------------------------------------------------------------------------- +// A READ-ONLY MIRROR of the +// switches as this process CACHED them, for persistent_topk_toggle_state(). +// +// Each switch below is a function-local static filled once by its lambda. The +// lambda now also stores the value it cached into the matching atomic here +// (publish_cached), and nothing else ever writes these atomics, so a mirror +// either says "not initialised yet" (-1) or holds exactly the cached value. +// The query only LOADS them: it never calls a getter, so asking cannot fill a +// cache from an environment that differs from the one the kernel saw. The +// parse expressions are unchanged; this is host code only. +// ------------------------------------------------------------------------- +std::atomic g_cached_tie_recent{-1}; +std::atomic g_cached_tie_det{-1}; +std::atomic g_cached_exact_medium{-1}; +std::atomic g_cached_det_large{-1}; +std::atomic g_cached_exact_decode{-1}; + +bool publish_cached(std::atomic& mirror, bool value) { + mirror.store(value ? 1 : 0, std::memory_order_release); + return value; +} + // MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above. // Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode, // most-recent-wins is +1.5 %. Both deterministic. Also the direction @@ -53,7 +76,9 @@ namespace { bool tie_recent_enabled() { static const bool enabled = [] { const char* raw = std::getenv("MERLIN_TIE_RECENT"); - return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false; + return publish_cached( + g_cached_tie_recent, + (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false); }(); return enabled; } @@ -61,7 +86,9 @@ bool tie_recent_enabled() { bool det_ties_enabled() { static const bool enabled = [] { const char* raw = std::getenv("MERLIN_TIE_DET"); - return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false; + return publish_cached( + g_cached_tie_det, + (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false); }(); return enabled; } @@ -69,7 +96,9 @@ bool det_ties_enabled() { bool exact_medium_enabled() { static const bool enabled = [] { const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM"); - return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0); + return publish_cached( + g_cached_exact_medium, + (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0)); }(); return enabled; } @@ -87,7 +116,9 @@ bool exact_medium_enabled() { bool det_large_enabled() { static const bool enabled = [] { const char* raw = std::getenv("MERLIN_TOPK_DET_LARGE"); - return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0); + return publish_cached( + g_cached_det_large, + (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0)); }(); return enabled; } @@ -104,7 +135,9 @@ bool det_large_enabled() { bool exact_decode_enabled() { static const bool enabled = [] { const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE"); - return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false; + return publish_cached( + g_cached_exact_decode, + (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false); }(); return enabled; } @@ -887,3 +920,49 @@ int64_t persistent_topk_exact_counters_slots() { return 0; #endif } + +// The persistent_topk +// switches AS THIS PROCESS CACHED THEM -- the values every launch of this +// process uses -- read from the mirrors above. Tensor-free, no CUDA call, no +// allocation, never initialises a cache (see publish_cached), so it is safe +// from anywhere and asking changes nothing. +// +// Packed into one int64 so the op needs no tensor and no dispatch key. +// 2 bits per field, field i at bits [2i, 2i+1]: 0 = off, 1 = on, +// 2 = the static has not been initialised in this process yet. +// field 0 MERLIN_TIE_DET (det_ties_enabled) +// field 1 MERLIN_TIE_RECENT as parsed (tie_recent_enabled) +// field 2 effective direction, 1 = most-recent (effective_tie_recent) +// field 3 MERLIN_TOPK_EXACT_MEDIUM (exact_medium_enabled) +// field 4 MERLIN_TOPK_EXACT_DECODE (exact_decode_enabled) +// field 5 MERLIN_TOPK_DET_LARGE (det_large_enabled) +// bits [48, 55] = field count (6), bits [56, 62] = layout version (1). A +// reader must check both and refuse any other layout rather than guess. +int64_t persistent_topk_toggle_state() { +#ifndef USE_ROCM + const auto field = [](int cached) -> int64_t { + return cached < 0 ? 2 : (cached != 0 ? 1 : 0); + }; + const int det = g_cached_tie_det.load(std::memory_order_acquire); + const int recent = g_cached_tie_recent.load(std::memory_order_acquire); + // The kernel's own rule, from the kernel's own header, on the cached values. + const int effective = + (det < 0 || recent < 0) + ? -1 + : (vllm::persistent::effective_tie_recent(det != 0, recent != 0) + ? 1 + : 0); + int64_t state = 0; + state |= field(det) << 0; + state |= field(recent) << 2; + state |= field(effective) << 4; + state |= field(g_cached_exact_medium.load(std::memory_order_acquire)) << 6; + state |= field(g_cached_exact_decode.load(std::memory_order_acquire)) << 8; + state |= field(g_cached_det_large.load(std::memory_order_acquire)) << 10; + state |= int64_t{6} << 48; + state |= int64_t{1} << 56; + return state; +#else + return 0; // layout version 0: no persistent_topk on ROCm +#endif +} diff --git a/csrc/libtorch_stable/torch_bindings.cpp b/csrc/libtorch_stable/torch_bindings.cpp index 344aa96faf..500e2fe55d 100644 --- a/csrc/libtorch_stable/torch_bindings.cpp +++ b/csrc/libtorch_stable/torch_bindings.cpp @@ -590,6 +590,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) { ops.def( "persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()"); ops.def("persistent_topk_exact_counters_slots() -> int"); + // v2.2 -- the switches as this process cached them (topk.cu for the layout). + ops.def("persistent_topk_toggle_state() -> int"); #ifdef VLLM_ENABLE_COOPERATIVE_TOPK ops.def( @@ -947,6 +949,9 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CompositeExplicitAutograd, ops) { // NotImplementedError "no fallback function is registered"). ops.impl("persistent_topk_exact_counters_slots", TORCH_BOX(&persistent_topk_exact_counters_slots)); + // Tensor-free for the same reason as the slot count above. + ops.impl("persistent_topk_toggle_state", + TORCH_BOX(&persistent_topk_toggle_state)); } // Cache ops diff --git a/vllm/v1/worker/e2_identity.py b/vllm/v1/worker/e2_identity.py new file mode 100644 index 0000000000..e457b09d74 --- /dev/null +++ b/vllm/v1/worker/e2_identity.py @@ -0,0 +1,148 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-rank launch-identity log line (log-only; changes no behaviour). + +After model loading -- and after the PLE offload placeholders are armed -- every +GPU worker rank logs one line per model role it holds:: + + e2-identity[PP

_TP]: ple_backend=@ ple_method= + ple_weight_dtype= breakable= compile_mode= + cudagraph_mode= role= + +(one line in the log). ``role=draft`` is logged only where a draft model is +loaded (the last PP stage). Every value is read from the live objects: + +* ``ple_backend``: the class of each PLE layer's ``ple_embedding`` module, and + the transport that serves its rows on this rank, read from the placeholder's + armed state -- ``host-mapped-pull`` (``_is_cpu_offloaded`` with a host output + pointer), ``ipc-push`` (offloaded, no host pointer) or ``local:`` (the + rank owns the table). +* ``ple_method``: the embedding's live ``quant_method``, or, on an offloaded + placeholder, the ``_offload_quant_method`` the rank dequantizes with. +* ``ple_weight_dtype``: the table's dtype where the rank owns it; on an + offloaded placeholder, the dtype of the armed transfer buffer, into which the + offload process copies the table's rows unchanged + (``get_offload_output_dtype``). +* ``breakable``: ``is_breakable_cudagraph_enabled()``, the switch the capture + path itself reads. +* ``compile_mode`` / ``cudagraph_mode``: the role's ``CompilationConfig`` + (config level, i.e. before KV-cache-time capture-mode resolution). + +A role with no PLE layer reports ``none`` for the three PLE fields; several +distinct values on one rank are joined with ``,``. +""" + +from typing import Any + +from torch import nn + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +_NONE = "none" + + +def _join(values: set[str]) -> str: + return ",".join(sorted(values)) if values else _NONE + + +def _enum_name(value: Any) -> str: + if value is None: + return _NONE + return str(getattr(value, "name", value)) + + +def _ple_embeddings(model: nn.Module | None) -> list[nn.Module]: + """The ``ple_embedding`` module of every PLE layer in ``model``.""" + if model is None: + return [] + found = [] + for module in model.modules(): + embedding = module.__dict__.get("_modules", {}).get("ple_embedding") + if isinstance(embedding, nn.Module): + found.append(embedding) + return found + + +def _transport(embedding: nn.Module) -> str: + if getattr(embedding, "_is_cpu_offloaded", False): + if getattr(embedding, "_host_output_ptr", None) is not None: + return "host-mapped-pull" + return "ipc-push" + weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None) + if weight is not None: + return f"local:{weight.device.type}" + return "local:unknown" + + +def _method(embedding: nn.Module) -> str: + table = getattr(embedding, "ngram_embedding", None) + method = getattr(table, "quant_method", None) + if method is None: + method = getattr(embedding, "_offload_quant_method", None) + return type(method).__name__ if method is not None else _NONE + + +def _weight_dtype(embedding: nn.Module) -> str: + weight = getattr(getattr(embedding, "ngram_embedding", None), "weight", None) + if weight is not None: + return str(weight.dtype) + if getattr(embedding, "_is_cpu_offloaded", False): + buffer = getattr(embedding, "_gpu_output_buffer", None) + if buffer is not None: + return str(buffer.dtype) + return "unknown" + + +def _rank_tag() -> str: + from vllm.distributed.parallel_state import get_pp_group, get_tp_group + + return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}" + + +def _draft(model_runner: Any) -> tuple[nn.Module | None, Any]: + """The loaded draft model and its config (V2 speculator or V1 drafter).""" + for name in ("speculator", "drafter"): + holder = getattr(model_runner, name, None) + model = getattr(holder, "model", None) + if isinstance(model, nn.Module): + return model, getattr(holder, "vllm_config", None) + return None, None + + +def identity_line( + tag: str, model: nn.Module | None, vllm_config: Any, role: str +) -> str: + from vllm.compilation.breakable_cudagraph import is_breakable_cudagraph_enabled + + embeddings = _ple_embeddings(model) + backends = {f"{type(e).__name__}@{_transport(e)}" for e in embeddings} + compilation = getattr(vllm_config, "compilation_config", None) + return ( + f"e2-identity[{tag}]: " + f"ple_backend={_join(backends)} " + f"ple_method={_join({_method(e) for e in embeddings})} " + f"ple_weight_dtype={_join({_weight_dtype(e) for e in embeddings})} " + f"breakable={'ON' if is_breakable_cudagraph_enabled() else 'OFF'} " + f"compile_mode={_enum_name(getattr(compilation, 'mode', None))} " + f"cudagraph_mode={_enum_name(getattr(compilation, 'cudagraph_mode', None))} " + f"role={role}" + ) + + +def log_launch_identity(model_runner: Any, vllm_config: Any) -> None: + """Log the identity line(s) for this rank. Never raises.""" + tag = "PP?_TP?" + try: + tag = _rank_tag() + roles = [("target", model_runner.get_model(), vllm_config)] + draft_model, draft_config = _draft(model_runner) + if draft_model is not None: + roles.append(("draft", draft_model, draft_config or vllm_config)) + for role, model, config in roles: + logger.info("%s", identity_line(tag, model, config, role)) + except Exception as error: # log-only: identity must never break a boot + logger.warning( + "e2-identity[%s]: unavailable (%s: %s)", tag, type(error).__name__, error + ) diff --git a/vllm/v1/worker/e2_toggles.py b/vllm/v1/worker/e2_toggles.py new file mode 100644 index 0000000000..0ae7ad2afc --- /dev/null +++ b/vllm/v1/worker/e2_toggles.py @@ -0,0 +1,444 @@ +# SPDX-License-Identifier: Apache-2.0 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project +"""Per-rank determinism-toggle line (log-only; changes no behaviour). + +Why this exists: the effective state of the +four determinism toggles and the three persistent_topk switches must be +attributable to a RANK and a PID. The kernel's own stderr line +(``[merlin] persistent_topk tie policy: ...``, topk.cu report_tie_policy_once) +is printed once per process with no tag, and FULL_K / QSA_SORT have no log +producer at all. Every GPU worker therefore logs exactly one line:: + + e2-toggles[PP

_TP]: pid= MERLIN_FULL_K= MERLIN_QSA_SORT= + MERLIN_TIE_DET= MERLIN_TIE_RECENT= MERLIN_TOPK_EXACT_MEDIUM= + MERLIN_TOPK_EXACT_DECODE= MERLIN_TOPK_DET_LARGE= + effective: full_k=<0|1> qsa_sort=<0|1|invalid> tie_det=<0|1> tie_recent=<0|1> + tie_recent_effective=<0|1> exact_medium=<0|1> exact_decode=<0|1> + det_large=<0|1> source= + cpp-cache: > + python: qsa_sort=<0|1|n-a> source=qsa_indexer._QSA_SORT + device: switch=<0|1|n-a> switch_large=<0|1|n-a> source=counter-slots-8-9 + +(one line in the log). ```` is ``unset``, a plain token, or a double-quoted +string with every byte outside ``[!-~]`` (and ``"`` / ``\\``) hex-escaped, so +the line always splits on whitespace. + +WHAT EACH SECTION IS +* ``effective:`` -- the environment at emission time, parsed with the SAME + rules the readers use (C++ ``std::getenv`` + ``std::atoi != 0`` for the six + C++ names, ``bool(int(...))`` for QSA_SORT; file:line of each below). The + C++ names are read through libc ``getenv`` -- the ``environ`` that + ``std::getenv`` reads -- not ``os.environ``, which can diverge from it after + an ``os.putenv``. A MIRROR of the parse, not the kernel's state. It exists + to report the rule; nothing may branch on it (vllm/envs.py "NOT A PYTHON + READ SITE" still holds: this module decides nothing). +* ``cpp-cache:`` -- the kernel's switches as the C++ statics cached them, from + ``torch.ops._C.persistent_topk_toggle_state`` and + ``torch.ops._moe_C.marlin_moe_full_k_state`` (query ops that only LOAD a + mirror the static initialisers publish; they never fill a cache). ``n-a`` + with a reason on a build without those ops -- the extensions this release + was first cut against (earlier v2.2 builds, not published) have none. +* ``python:`` -- ``qsa_indexer._QSA_SORT``, the module constant the QSA sort + actually branches on (looked up in ``sys.modules``, never imported here; + ``n-a`` if the module was never loaded in this process). +* ``device:`` -- counter slots [8]/[9], which the counter op fills from the + C++ ``exact_medium_enabled()`` / ``det_large_enabled()`` statics + (topk.cu persistent_topk_exact_counters). + +WHEN. Once per worker, after ``compile_or_warm_up_model`` has run the MRV2 +kernel warmup and cudagraph capture (``arm()``), i.e. after this process's +first persistent_topk and Marlin MoE launches -- NOT at the counter reader's +first read: an integration boot logged its first counter line BEFORE +the first tie-policy line in the engine log, because the +first ``maybe_log`` runs at the head of the warmup's first execute step, before +that step's forward. When the cpp-cache ops exist and a static is still +uninitialised at ``arm()``, the line is deferred to the head of the first +execute step where every static is filled, and forced out at shutdown +otherwise (with ``uninit`` values, which the smoke FAILs). It does not depend on +``MERLIN_TOPK_EXACT_COUNTERS_LOG``: it fires with the counter log off. + +WHAT THIS DOES NOT PROVE. Without the cpp-cache ops the C++ state is inferred, +not read: for FULL_K, TIE_DET, TIE_RECENT and EXACT_DECODE the line is the +environment at emission time run through the C++ parser's rules, and the only +C++-derived evidence is ``device:`` (EXACT_MEDIUM and DET_LARGE) plus the +untagged stderr policy line. The residual gap is an environment mutated +between the kernel's first ``getenv`` and this read. Nothing in this tree does +that (by grep of csrc/): no ``setenv``/``putenv``/``unsetenv`` under csrc/, and +the only runtime ``os.environ.update`` calls in vllm/ are the RunAI streamer's +fixed ``RUNAI_STREAMER_*`` keys (model_executor/model_loader/ +runai_streamer_loader.py:73) and the Ray executor's ``override_env_vars`` +(v1/executor/ray_utils.py:180), which the multiproc executor never calls. With the +cpp-cache ops that gap closes for every C++ name. Even then the line proves the +CONFIGURATION each rank ran, not that a switch changed any output: that is the +job of an output-equality run (identical sends -> one distinct output). +""" + +from __future__ import annotations + +import os +import sys +from collections.abc import Callable + +from vllm.logger import init_logger + +logger = init_logger(__name__) + +# (environment name, effective key, value when unset or empty) -- the C++ +# readers, each `static const bool enabled = [] { ... }();` once per process: +# MERLIN_FULL_K csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu:57-63 +# MERLIN_TIE_DET csrc/libtorch_stable/topk.cu:60-66 +# MERLIN_TIE_RECENT csrc/libtorch_stable/topk.cu:52-58 +# MERLIN_TOPK_EXACT_MEDIUM csrc/libtorch_stable/topk.cu:68-74 (unset -> ON) +# MERLIN_TOPK_EXACT_DECODE csrc/libtorch_stable/topk.cu:103-109 +# MERLIN_TOPK_DET_LARGE csrc/libtorch_stable/topk.cu:86-92 (unset -> ON) +CXX_SWITCHES: tuple[tuple[str, str, bool], ...] = ( + ("MERLIN_FULL_K", "full_k", False), + ("MERLIN_TIE_DET", "tie_det", False), + ("MERLIN_TIE_RECENT", "tie_recent", False), + ("MERLIN_TOPK_EXACT_MEDIUM", "exact_medium", True), + ("MERLIN_TOPK_EXACT_DECODE", "exact_decode", False), + ("MERLIN_TOPK_DET_LARGE", "det_large", True), +) +QSA_SORT_NAME = "MERLIN_QSA_SORT" +RAW_ORDER = ( + "MERLIN_FULL_K", + "MERLIN_QSA_SORT", + "MERLIN_TIE_DET", + "MERLIN_TIE_RECENT", + "MERLIN_TOPK_EXACT_MEDIUM", + "MERLIN_TOPK_EXACT_DECODE", + "MERLIN_TOPK_DET_LARGE", +) +EFFECTIVE_ORDER = ( + "full_k", + "qsa_sort", + "tie_det", + "tie_recent", + "tie_recent_effective", + "exact_medium", + "exact_decode", + "det_large", +) +CPP_ORDER = tuple(k for k in EFFECTIVE_ORDER if k != "qsa_sort") + +_QSA_MODULE = "vllm.models.qwen4_exp.nvidia.ops.qsa_indexer" +_C_SPACE = b" \t\n\v\f\r" # isspace() in the "C" locale +_LONG_MIN, _LONG_MAX = -(2**63), 2**63 - 1 # LP64 long, the reference host's glibc + + +# --------------------------------------------------------------- parsers ---- +def c_atoi(raw: bytes) -> int: + """glibc ``atoi``: ``(int) strtol(raw, NULL, 10)`` (glibc stdlib/atoi.c). + + Leading C-locale whitespace, one optional sign, then decimal digits up to + the first non-digit; no digits -> 0. ``strtol`` clamps to the ``long`` + range and the ``(int)`` conversion keeps the low 32 bits (GCC's documented + modulo behaviour). So ``"true"`` -> 0, ``"1abc"`` -> 1, ``"0x1"`` -> 0, + ``" +1"`` -> 1, ``"+ 1"`` -> 0 and ``"4294967296"`` -> 0. + """ + i, n = 0, len(raw) + while i < n and raw[i] in _C_SPACE: + i += 1 + negative = False + if i < n and raw[i] in b"+-": + negative = raw[i] == ord("-") + i += 1 + value, digits = 0, 0 + while i < n and 0x30 <= raw[i] <= 0x39: + value = value * 10 + (raw[i] - 0x30) + digits += 1 + i += 1 + if digits == 0: + return 0 + if negative: + value = -value + value = max(_LONG_MIN, min(_LONG_MAX, value)) + value &= 0xFFFFFFFF + return value - (1 << 32) if value >= (1 << 31) else value + + +def cxx_switch(raw: bytes | None, unset_value: bool) -> bool: + """The C++ getter body, both shapes: + default OFF ``(raw != nullptr && raw[0] != '\\0') ? (atoi(raw) != 0) : false`` + default ON ``(raw == nullptr || raw[0] == '\\0') ? true : (atoi(raw) != 0)`` + An environment value cannot contain NUL, so "empty" is exactly + ``raw[0] == '\\0'``.""" + if raw is None or raw == b"": + return unset_value + return c_atoi(raw) != 0 + + +def qsa_sort_value(raw: str | None) -> str: + """vllm/envs.py:2372 ``bool(int(os.getenv("MERLIN_QSA_SORT", "0")))``, + read once at import by qsa_indexer.py:51 (``_QSA_SORT``). ``int()`` raises + on ``"true"`` and on ``""`` -- that engine would not have booted -- which is + reported as ``invalid`` rather than guessed. Python's own ``int`` is used, + so whitespace, ``+``/``-``, ``1_0`` and non-ASCII digits parse exactly as + the reader parses them.""" + try: + return "1" if bool(int("0" if raw is None else raw)) else "0" + except ValueError: + return "invalid" + + +def effective_tie_recent(det_ties: bool, tie_recent: bool) -> bool: + """csrc/libtorch_stable/persistent_topk.cuh:65-68: ``det_ties ? tie_recent + : true`` -- with TIE_DET off the restart paths keep most-recent-wins.""" + return tie_recent if det_ties else True + + +# --------------------------------------------------------------- sources ---- +def _libc_getenv() -> tuple[Callable[[str], bytes | None], str]: + """libc ``getenv``: the ``environ`` the C++ ``std::getenv`` calls read. + + Falls back to ``os.environb`` / ``os.environ`` -- and says so in the line's + ``source=`` -- where there is no C library to ask (an offline run off Linux). + """ + try: + import ctypes + + getenv = ctypes.CDLL(None).getenv + getenv.restype = ctypes.c_char_p + getenv.argtypes = [ctypes.c_char_p] + + def read(name: str) -> bytes | None: + return getenv(name.encode("ascii")) + + return read, "libc-getenv" + except (OSError, AttributeError, TypeError): + pass + environb = getattr(os, "environb", None) + if environb is not None: + return (lambda name: environb.get(name.encode("ascii"))), "os-environ" + + def read_environ(name: str) -> bytes | None: + value = os.environ.get(name) + return None if value is None else value.encode("utf-8", "surrogateescape") + + return read_environ, "os-environ" + + +def _printable(b: int) -> bool: + return 0x21 <= b <= 0x7E and b not in b'"\\' + + +def encode_raw(raw: bytes | None) -> str: + """``unset``, a plain token, or a quoted, hex-escaped string -- never a + space, so the line splits on whitespace. The literal value ``unset`` is + quoted so it cannot pass for an absent variable.""" + if raw is None: + return "unset" + if raw and raw != b"unset" and all(_printable(b) for b in raw): + return raw.decode("ascii") + return '"' + "".join(chr(b) if _printable(b) else f"\\x{b:02x}" for b in raw) + '"' + + +def environ_view( + cxx_getenv: Callable[[str], bytes | None], + py_getenv: Callable[[str], str | None], +) -> tuple[dict[str, str], dict[str, str]]: + """(raw value by name, effective map) from one read of the environment.""" + raw: dict[str, str] = {} + flags: dict[str, bool] = {} + for name, key, unset_value in CXX_SWITCHES: + value = cxx_getenv(name) + raw[name] = encode_raw(value) + flags[key] = cxx_switch(value, unset_value) + qsa_raw = py_getenv(QSA_SORT_NAME) + raw[QSA_SORT_NAME] = encode_raw( + None if qsa_raw is None else qsa_raw.encode("utf-8", "surrogateescape") + ) + eff = {key: "1" if flags[key] else "0" for key in flags} + eff["qsa_sort"] = qsa_sort_value(qsa_raw) + eff["tie_recent_effective"] = ( + "1" if effective_tie_recent(flags["tie_det"], flags["tie_recent"]) else "0" + ) + return raw, {key: eff[key] for key in EFFECTIVE_ORDER} + + +_TOPK_STATE_FIELDS = ( + "tie_det", + "tie_recent", + "tie_recent_effective", + "exact_medium", + "exact_decode", + "det_large", +) + + +def decode_topk_state(state: int) -> dict[str, str] | str: + """Unpack ``persistent_topk_toggle_state()``: 2 bits per field (0 off, + 1 on, 2 not initialised), field count at bits 48-55 (6), layout version at + bits 56-62 (1). Any other layout is refused with a reason string.""" + version, count = (state >> 56) & 0x7F, (state >> 48) & 0xFF + if version != 1 or count != len(_TOPK_STATE_FIELDS): + return f"unknown-layout-v{version}-{count}-fields" + names = {0: "0", 1: "1", 2: "uninit"} + return { + key: names.get((state >> (2 * i)) & 3, "invalid") + for i, key in enumerate(_TOPK_STATE_FIELDS) + } + + +def decode_full_k_state(state: int) -> str: + """``marlin_moe_full_k_state()``: -1 not initialised, 0 off, 1 on.""" + return {-1: "uninit", 0: "0", 1: "1"}.get(int(state), "invalid") + + +def _op(namespace: str, name: str): + try: + import torch + + return getattr(getattr(torch.ops, namespace), name, None) + except (AttributeError, RuntimeError, ImportError): + return None + + +def cpp_cache() -> dict[str, str] | str: + """The C++ cached switches, or a reason string when the build cannot say.""" + topk_op = _op("_C", "persistent_topk_toggle_state") + full_k_op = _op("_moe_C", "marlin_moe_full_k_state") + missing = [ + name + for name, op in ( + ("_C.persistent_topk_toggle_state", topk_op), + ("_moe_C.marlin_moe_full_k_state", full_k_op), + ) + if op is None + ] + if missing: + return "op-absent:" + ",".join(missing) + decoded = decode_topk_state(int(topk_op())) + if isinstance(decoded, str): + return decoded + out = dict(decoded, full_k=decode_full_k_state(int(full_k_op()))) + return {key: out[key] for key in CPP_ORDER} + + +def qsa_sort_live() -> str: + module = sys.modules.get(_QSA_MODULE) + if module is None or not hasattr(module, "_QSA_SORT"): + return "n-a" + return "1" if bool(module._QSA_SORT) else "0" + + +def device_switches() -> dict[str, str]: + """Counter slots [8]/[9] (switch / switch_large). One synchronizing read + (topk_counters.read refuses inside a capture), so it is only called from + arm() and the head of an execute step.""" + try: + from vllm.models.qwen4_exp.nvidia.ops import topk_counters + + counters = topk_counters.read() + except Exception: + counters = {} + return { + key: str(counters[key]) if key in counters else "n-a" + for key in ("switch", "switch_large") + } + + +# ------------------------------------------------------------------ line ---- +def toggles_line( + tag: str, + pid: int, + raw: dict[str, str], + effective: dict[str, str], + env_source: str, + cpp: dict[str, str] | str, + qsa_live: str, + device: dict[str, str], +) -> str: + parts = [f"e2-toggles[{tag}]: pid={pid}"] + parts += [f"{name}={raw[name]}" for name in RAW_ORDER] + parts.append("effective:") + parts += [f"{key}={effective[key]}" for key in EFFECTIVE_ORDER] + parts.append(f"source={env_source}") + parts.append("cpp-cache:") + if isinstance(cpp, str): + parts += ["n-a", f"reason={cpp}"] + else: + parts += [f"{key}={cpp[key]}" for key in CPP_ORDER] + parts.append("source=cpp-cache") + parts += ["python:", f"qsa_sort={qsa_live}", "source=qsa_indexer._QSA_SORT"] + parts.append("device:") + parts += [f"{key}={device[key]}" for key in ("switch", "switch_large")] + parts.append("source=counter-slots-8-9") + return " ".join(parts) + + +def _rank_tag() -> str: + from vllm.distributed.parallel_state import get_pp_group, get_tp_group + + return f"PP{get_pp_group().rank_in_group}_TP{get_tp_group().rank_in_group}" + + +# ----------------------------------------------------------------- state ---- +_armed = False +_emitted = False +_deferral_noted = False + + +def _reset_for_tests() -> None: + global _armed, _emitted, _deferral_noted + _armed = _emitted = _deferral_noted = False + + +def arm(**kwargs) -> str: + """Once per worker, at the end of compile_or_warm_up_model (warmup and + capture done): emit now, unless a C++ static is still uninitialised.""" + global _armed + _armed = True + return maybe_emit(**kwargs) + + +def maybe_emit( + force: bool = False, + *, + tag_fn: Callable[[], str] = _rank_tag, + getenv_fn: Callable[[], tuple[Callable[[str], bytes | None], str]] = _libc_getenv, + cpp_fn: Callable[[], dict[str, str] | str] = cpp_cache, + device_fn: Callable[[], dict[str, str]] = device_switches, + qsa_fn: Callable[[], str] = qsa_sort_live, +) -> str: + """Emit the line at most once per process. One bool check when already + emitted or not yet armed, so it can sit at the head of every execute step. + Never raises. Returns the line when this call emitted it, else "".""" + global _emitted, _deferral_noted + if _emitted or not _armed: + return "" + tag = "PP?_TP?" + try: + tag = tag_fn() + cpp = cpp_fn() + pending = ( + sorted(k for k, v in cpp.items() if v == "uninit") + if isinstance(cpp, dict) + else [] + ) + if pending and not force: + if not _deferral_noted: + _deferral_noted = True + # Deliberately not 'e2-toggles[': the smoke counts those. + logger.info( + "e2-toggles deferred on %s: C++ cache not initialised for %s", + tag, + ",".join(pending), + ) + return "" + cxx_getenv, env_source = getenv_fn() + raw, effective = environ_view(cxx_getenv, os.environ.get) + line = toggles_line( + tag, os.getpid(), raw, effective, env_source, cpp, qsa_fn(), device_fn() + ) + _emitted = True + logger.info("%s", line) + return line + except Exception as error: # log-only: must never break a boot or a step + _emitted = True + logger.warning( + "e2-toggles[%s]: unavailable (%s: %s)", tag, type(error).__name__, error + ) + return "" diff --git a/vllm/v1/worker/gpu/model_runner.py b/vllm/v1/worker/gpu/model_runner.py index a9d87e4d2b..374a1df853 100644 --- a/vllm/v1/worker/gpu/model_runner.py +++ b/vllm/v1/worker/gpu/model_runner.py @@ -85,7 +85,7 @@ from vllm.v1.watermarking.spec_decode import ( create_speculative_target_watermarker, speculative_target_watermark_key, ) -from vllm.v1.worker import guard_telemetry +from vllm.v1.worker import e2_toggles, guard_telemetry from vllm.v1.worker.block_table import get_block_table_width from vllm.v1.worker.cp_utils import check_attention_cp_compatibility from vllm.v1.worker.gpu import pcp_manager as pcp @@ -1863,6 +1863,9 @@ class GPUModelRunner(LoRAModelRunnerMixin): # never fire on them. Outside any capture: this is the engine # thread between steps (read() re-checks). topk_counters.maybe_log() + # v2.2: the e2-toggles line if arm() had to defer it (a C++ static + # still uninitialised). One bool check once it has been emitted. + e2_toggles.maybe_emit() # Update the request states. self.update_pp_decode_requests() self.finish_requests(scheduler_output) @@ -2580,6 +2583,8 @@ class GPUModelRunner(LoRAModelRunnerMixin): # The last window of a run is the one a cadence log always # misses; emit it before the device state goes away. topk_counters.maybe_log(force=True) + # v2.2: a still-deferred e2-toggles line goes out now, 'uninit' and all. + e2_toggles.maybe_emit(force=True) if self._ple_offload_connector is not None: self._ple_offload_connector.close() self._ple_offload_connector = None diff --git a/vllm/v1/worker/gpu_worker.py b/vllm/v1/worker/gpu_worker.py index 74c970594d..44732ced1f 100644 --- a/vllm/v1/worker/gpu_worker.py +++ b/vllm/v1/worker/gpu_worker.py @@ -740,6 +740,12 @@ class Worker(WorkerBase): model_runner = cast("GPUModelRunnerV2", self.model_runner) model_runner._setup_ple_offload(self.parallel_config._ple_offload_ipc_path) + # Log-only: this rank's PLE / capture identity for target and drafter, + # read after the offload placeholders are armed (v1/worker/e2_identity.py). + from vllm.v1.worker.e2_identity import log_launch_identity + + log_launch_identity(self.model_runner, self.vllm_config) + if has_ec_transfer(): get_ec_transfer().start_worker_services() @@ -1093,6 +1099,13 @@ class Worker(WorkerBase): with self._get_cudagraph_capture_context(): cuda_graph_memory_bytes = self.model_runner.capture_model() + # Log-only: this rank's determinism toggles, once, now that the warmup + # and capture above have launched persistent_topk and Marlin MoE, so + # their C++ statics are filled (v1/worker/e2_toggles.py). Never raises. + from vllm.v1.worker import e2_toggles + + e2_toggles.arm() + # Compare actual vs estimated CUDA graph memory (if we did profiling) if ( hasattr(self, "cudagraph_memory_estimate")