[TopK] Diagnostic-only build defines and qualification test cells

Compile-time defines (default 0, device code unchanged by default) for the
delayed-publication stress, the pre-repair control and a forced-fault cell;
test cells for the OFF-arm controls, the QSA_SORT contract, FULL_K attribution
on the Marlin MoE GEMM and legacy-tail repeated-row membership.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:56:00 +07:00
co-authored by Hinata
parent cff46af086
commit bef464a69d
2 changed files with 110 additions and 0 deletions
+85
View File
@@ -685,6 +685,74 @@ __device__ __noinline__ bool histogram_2048_topk(
#define MERLIN_TOPK_EXACT_TRAP 1
#endif
// ============================================================================
// DIAGNOSTIC BUILDS ONLY -- never set in a release or serving build.
//
// Three compile-time defines exist so the reference host can run the two large-path
// diagnostic cells. Every one defaults to 0 and every use
// sits behind `#if`, so the default build's device code is identical to the
// tree without them (offline, real TU, sm_86, against an earlier commit: SASS identical,
// cubin identical after masking nvcc's per-compile internal-linkage TU hash,
// which differs even between two compiles of the SAME source -- see the
// diagnostic-build notes). A build carrying any of them prints a
// `[merlin] persistent_topk DIAGNOSTIC BUILD` line on its first launch
// (topk.cu), which the smoke's startup-warning scan must reject.
//
// MERLIN_TOPK_DIAG_DELAY=<cycles>
// In `radix_topk`'s round loop, every NON-LEADER warp (warp != 0) of
// every NON-LEADER CTA (cta_in_group != 0) spins <cycles> clock64 ticks
// before merging its shared bins into the group's global histogram.
// Warp 0 holds thread 0, the arrival publisher, so it is never delayed:
// the delay opens exactly the window the histogram-publication barrier closes -- a
// publisher that announces arrival before its own CTA's other warps have
// produced. CTA 0 is never delayed, so it reads the histogram as soon
// as the arrival count completes.
// MERLIN_TOPK_DIAG_PREREPAIR=1
// Compile out the histogram-publication barrier. This is the
// PRE-REPAIR control for the delayed-publication stress: with a delay it
// is expected to return wrong or faulted rows; the repaired build with
// the same delay must not. Only meaningful with MERLIN_TOPK_DIAG_DELAY.
// Do NOT run it with MERLIN_TOPK_DET_LARGE=0: the legacy tail bounds no
// emission slot, so a disagreeing pivot writes out of bounds there.
// MERLIN_TOPK_DIAG_FORCE_FAULT=1 (+ MERLIN_TOPK_DIAG_FORCE_FAULT_CTA, 1)
// Force the CTA-local X3 predicate in that CTA on every deterministic
// large row, exactly as MERLIN_TOPK_LARGE_FAULT_INJECT does -- but WITHOUT
// suppressing `__trap()`. The trap-ON fault arm: the process's CUDA
// context dies. Run only in a disposable subprocess.
// ============================================================================
#ifndef MERLIN_TOPK_DIAG_DELAY
#define MERLIN_TOPK_DIAG_DELAY 0
#endif
#ifndef MERLIN_TOPK_DIAG_PREREPAIR
#define MERLIN_TOPK_DIAG_PREREPAIR 0
#endif
#ifndef MERLIN_TOPK_DIAG_FORCE_FAULT
#define MERLIN_TOPK_DIAG_FORCE_FAULT 0
#endif
#ifndef MERLIN_TOPK_DIAG_FORCE_FAULT_CTA
#define MERLIN_TOPK_DIAG_FORCE_FAULT_CTA 1
#endif
#define MERLIN_TOPK_DIAG_ANY \
((MERLIN_TOPK_DIAG_DELAY) != 0 || (MERLIN_TOPK_DIAG_PREREPAIR) != 0 || \
(MERLIN_TOPK_DIAG_FORCE_FAULT) != 0)
#if MERLIN_TOPK_DIAG_FORCE_FAULT && !MERLIN_TOPK_EXACT_TRAP
#error "MERLIN_TOPK_DIAG_FORCE_FAULT is the trap-ON cell; it needs MERLIN_TOPK_EXACT_TRAP=1"
#endif
#if MERLIN_TOPK_DIAG_FORCE_FAULT && (MERLIN_TOPK_DIAG_DELAY || MERLIN_TOPK_DIAG_PREREPAIR)
#error "build the trap-ON and the delayed-publication diagnostics separately"
#endif
#if MERLIN_TOPK_DIAG_DELAY
// Busy-wait on the SM clock. `volatile` sink so the loop is not elided.
__device__ __noinline__ void merlin_topk_diag_spin(long long cycles) {
const long long start = clock64();
volatile long long now = start;
while (now - start < cycles) {
now = clock64();
}
}
#endif
// Exact, bounded-memory re-selection for a single medium-path row.
//
// `scores` is already offset to the row. Every radix round rescans the whole
@@ -1621,6 +1689,13 @@ __device__ __noinline__ void radix_emit_deterministic(
static_cast<int>(cta_in_group) == fault_inject_cta) {
my_keep_checked = local_eq + 1u;
}
#if MERLIN_TOPK_DIAG_FORCE_FAULT
// DIAGNOSTIC BUILD ONLY: the same CTA-local predicate, forced at compile
// time, so the trap below is NOT suppressed (fault_inject_cta stays -1).
if (cta_in_group == static_cast<uint32_t>(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA)) {
my_keep_checked = local_eq + 1u;
}
#endif
if (my_keep_checked > local_eq) fault = 1u;
if (gt_base + eq_keep_base + local_gt + my_keep >
static_cast<uint32_t>(TopK)) {
@@ -1954,6 +2029,14 @@ __device__ void radix_topk(const float* __restrict__ row_input,
}
__syncthreads();
#if MERLIN_TOPK_DIAG_DELAY
// DIAGNOSTIC BUILD ONLY (see the define block): hold back the producers
// that are NOT the arrival publisher, in every CTA that is not the leader.
if (cta_in_group != 0u && (tx >> 5) != 0u) {
merlin_topk_diag_spin(static_cast<long long>(MERLIN_TOPK_DIAG_DELAY));
}
#endif
for (uint32_t i = tx; i < RADIX; i += kThreadsPerBlock) {
if (local_histogram[i] > 0) {
atomicAdd(&current_hist[i], local_histogram[i]);
@@ -1976,7 +2059,9 @@ __device__ void radix_topk(const float* __restrict__ row_input,
// nothing here -- and reads a half-written histogram, which changes the
// pivot. The barrier must be between the producing stores and the leader's
// publication; program order in one thread is not cross-thread ordering.
#if !MERLIN_TOPK_DIAG_PREREPAIR
__syncthreads();
#endif
if (tx == 0) {
red_release(&state->arrival_counter, 1);
+25
View File
@@ -202,6 +202,28 @@ void report_tie_policy_once() {
(void)reported;
}
#if MERLIN_TOPK_DIAG_ANY
// DIAGNOSTIC BUILD ONLY (persistent_topk.cuh, "DIAGNOSTIC BUILDS ONLY"). Say
// so once per process, on stderr, before the first launch: the diagnostic-build tests
// key on this line to prove which build they ran against, and the smoke's
// startup-warning scan must reject any process that prints it.
void report_diag_build_once() {
static const bool reported = [] {
std::fprintf(stderr,
"[merlin] persistent_topk DIAGNOSTIC BUILD: "
"MERLIN_TOPK_DIAG_DELAY=%lld MERLIN_TOPK_DIAG_PREREPAIR=%d "
"MERLIN_TOPK_DIAG_FORCE_FAULT=%d "
"MERLIN_TOPK_DIAG_FORCE_FAULT_CTA=%d -- NOT FOR SERVING\n",
static_cast<long long>(MERLIN_TOPK_DIAG_DELAY),
static_cast<int>(MERLIN_TOPK_DIAG_PREREPAIR),
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT),
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA));
return true;
}();
(void)reported;
}
#endif
// Process-lifetime device counters, one block per device. The kernel only
// ever atomicAdds into them, so the launch path needs no host sync and the
// values accumulate across cudagraph replays.
@@ -568,6 +590,9 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
params.tie_recent = tie_recent_enabled();
params.exact_decode = exact_decode_enabled();
report_tie_policy_once();
#if MERLIN_TOPK_DIAG_ANY
report_diag_build_once();
#endif
// Allocate only off the capture path; in the ordinary engine lifecycle the
// eager warmup lands here first, long before capture. The explicit,
// quiescent-phase entry point is the reader op.