diff --git a/csrc/libtorch_stable/persistent_topk.cuh b/csrc/libtorch_stable/persistent_topk.cuh index 504db92a92..cb6443c9a1 100644 --- a/csrc/libtorch_stable/persistent_topk.cuh +++ b/csrc/libtorch_stable/persistent_topk.cuh @@ -685,6 +685,74 @@ __device__ __noinline__ bool histogram_2048_topk( #define MERLIN_TOPK_EXACT_TRAP 1 #endif +// ============================================================================ +// DIAGNOSTIC BUILDS ONLY -- never set in a release or serving build. +// +// Three compile-time defines exist so the reference host can run the two large-path +// diagnostic cells. Every one defaults to 0 and every use +// sits behind `#if`, so the default build's device code is identical to the +// tree without them (offline, real TU, sm_86, against an earlier commit: SASS identical, +// cubin identical after masking nvcc's per-compile internal-linkage TU hash, +// which differs even between two compiles of the SAME source -- see the +// diagnostic-build notes). A build carrying any of them prints a +// `[merlin] persistent_topk DIAGNOSTIC BUILD` line on its first launch +// (topk.cu), which the smoke's startup-warning scan must reject. +// +// MERLIN_TOPK_DIAG_DELAY= +// In `radix_topk`'s round loop, every NON-LEADER warp (warp != 0) of +// every NON-LEADER CTA (cta_in_group != 0) spins clock64 ticks +// before merging its shared bins into the group's global histogram. +// Warp 0 holds thread 0, the arrival publisher, so it is never delayed: +// the delay opens exactly the window the histogram-publication barrier closes -- a +// publisher that announces arrival before its own CTA's other warps have +// produced. CTA 0 is never delayed, so it reads the histogram as soon +// as the arrival count completes. +// MERLIN_TOPK_DIAG_PREREPAIR=1 +// Compile out the histogram-publication barrier. This is the +// PRE-REPAIR control for the delayed-publication stress: with a delay it +// is expected to return wrong or faulted rows; the repaired build with +// the same delay must not. Only meaningful with MERLIN_TOPK_DIAG_DELAY. +// Do NOT run it with MERLIN_TOPK_DET_LARGE=0: the legacy tail bounds no +// emission slot, so a disagreeing pivot writes out of bounds there. +// MERLIN_TOPK_DIAG_FORCE_FAULT=1 (+ MERLIN_TOPK_DIAG_FORCE_FAULT_CTA, 1) +// Force the CTA-local X3 predicate in that CTA on every deterministic +// large row, exactly as MERLIN_TOPK_LARGE_FAULT_INJECT does -- but WITHOUT +// suppressing `__trap()`. The trap-ON fault arm: the process's CUDA +// context dies. Run only in a disposable subprocess. +// ============================================================================ +#ifndef MERLIN_TOPK_DIAG_DELAY + #define MERLIN_TOPK_DIAG_DELAY 0 +#endif +#ifndef MERLIN_TOPK_DIAG_PREREPAIR + #define MERLIN_TOPK_DIAG_PREREPAIR 0 +#endif +#ifndef MERLIN_TOPK_DIAG_FORCE_FAULT + #define MERLIN_TOPK_DIAG_FORCE_FAULT 0 +#endif +#ifndef MERLIN_TOPK_DIAG_FORCE_FAULT_CTA + #define MERLIN_TOPK_DIAG_FORCE_FAULT_CTA 1 +#endif +#define MERLIN_TOPK_DIAG_ANY \ + ((MERLIN_TOPK_DIAG_DELAY) != 0 || (MERLIN_TOPK_DIAG_PREREPAIR) != 0 || \ + (MERLIN_TOPK_DIAG_FORCE_FAULT) != 0) +#if MERLIN_TOPK_DIAG_FORCE_FAULT && !MERLIN_TOPK_EXACT_TRAP + #error "MERLIN_TOPK_DIAG_FORCE_FAULT is the trap-ON cell; it needs MERLIN_TOPK_EXACT_TRAP=1" +#endif +#if MERLIN_TOPK_DIAG_FORCE_FAULT && (MERLIN_TOPK_DIAG_DELAY || MERLIN_TOPK_DIAG_PREREPAIR) + #error "build the trap-ON and the delayed-publication diagnostics separately" +#endif + +#if MERLIN_TOPK_DIAG_DELAY +// Busy-wait on the SM clock. `volatile` sink so the loop is not elided. +__device__ __noinline__ void merlin_topk_diag_spin(long long cycles) { + const long long start = clock64(); + volatile long long now = start; + while (now - start < cycles) { + now = clock64(); + } +} +#endif + // Exact, bounded-memory re-selection for a single medium-path row. // // `scores` is already offset to the row. Every radix round rescans the whole @@ -1621,6 +1689,13 @@ __device__ __noinline__ void radix_emit_deterministic( static_cast(cta_in_group) == fault_inject_cta) { my_keep_checked = local_eq + 1u; } +#if MERLIN_TOPK_DIAG_FORCE_FAULT + // DIAGNOSTIC BUILD ONLY: the same CTA-local predicate, forced at compile + // time, so the trap below is NOT suppressed (fault_inject_cta stays -1). + if (cta_in_group == static_cast(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA)) { + my_keep_checked = local_eq + 1u; + } +#endif if (my_keep_checked > local_eq) fault = 1u; if (gt_base + eq_keep_base + local_gt + my_keep > static_cast(TopK)) { @@ -1954,6 +2029,14 @@ __device__ void radix_topk(const float* __restrict__ row_input, } __syncthreads(); +#if MERLIN_TOPK_DIAG_DELAY + // DIAGNOSTIC BUILD ONLY (see the define block): hold back the producers + // that are NOT the arrival publisher, in every CTA that is not the leader. + if (cta_in_group != 0u && (tx >> 5) != 0u) { + merlin_topk_diag_spin(static_cast(MERLIN_TOPK_DIAG_DELAY)); + } +#endif + for (uint32_t i = tx; i < RADIX; i += kThreadsPerBlock) { if (local_histogram[i] > 0) { atomicAdd(¤t_hist[i], local_histogram[i]); @@ -1976,7 +2059,9 @@ __device__ void radix_topk(const float* __restrict__ row_input, // nothing here -- and reads a half-written histogram, which changes the // pivot. The barrier must be between the producing stores and the leader's // publication; program order in one thread is not cross-thread ordering. +#if !MERLIN_TOPK_DIAG_PREREPAIR __syncthreads(); +#endif if (tx == 0) { red_release(&state->arrival_counter, 1); diff --git a/csrc/libtorch_stable/topk.cu b/csrc/libtorch_stable/topk.cu index fed5fc885a..c21524ee96 100644 --- a/csrc/libtorch_stable/topk.cu +++ b/csrc/libtorch_stable/topk.cu @@ -202,6 +202,28 @@ void report_tie_policy_once() { (void)reported; } +#if MERLIN_TOPK_DIAG_ANY +// DIAGNOSTIC BUILD ONLY (persistent_topk.cuh, "DIAGNOSTIC BUILDS ONLY"). Say +// so once per process, on stderr, before the first launch: the diagnostic-build tests +// key on this line to prove which build they ran against, and the smoke's +// startup-warning scan must reject any process that prints it. +void report_diag_build_once() { + static const bool reported = [] { + std::fprintf(stderr, + "[merlin] persistent_topk DIAGNOSTIC BUILD: " + "MERLIN_TOPK_DIAG_DELAY=%lld MERLIN_TOPK_DIAG_PREREPAIR=%d " + "MERLIN_TOPK_DIAG_FORCE_FAULT=%d " + "MERLIN_TOPK_DIAG_FORCE_FAULT_CTA=%d -- NOT FOR SERVING\n", + static_cast(MERLIN_TOPK_DIAG_DELAY), + static_cast(MERLIN_TOPK_DIAG_PREREPAIR), + static_cast(MERLIN_TOPK_DIAG_FORCE_FAULT), + static_cast(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA)); + return true; + }(); + (void)reported; +} +#endif + // Process-lifetime device counters, one block per device. The kernel only // ever atomicAdds into them, so the launch path needs no host sync and the // values accumulate across cudagraph replays. @@ -568,6 +590,9 @@ void launch_persistent_topk(const torch::stable::Tensor& logits, params.tie_recent = tie_recent_enabled(); params.exact_decode = exact_decode_enabled(); report_tie_policy_once(); +#if MERLIN_TOPK_DIAG_ANY + report_diag_build_once(); +#endif // Allocate only off the capture path; in the ordinary engine lifecycle the // eager warmup lands here first, long before capture. The explicit, // quiescent-phase entry point is the reader op.