[TopK] Diagnostic-only build defines and qualification test cells
Compile-time defines (default 0, device code unchanged by default) for the delayed-publication stress, the pre-repair control and a forced-fault cell; test cells for the OFF-arm controls, the QSA_SORT contract, FULL_K attribution on the Marlin MoE GEMM and legacy-tail repeated-row membership. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -685,6 +685,74 @@ __device__ __noinline__ bool histogram_2048_topk(
|
||||
#define MERLIN_TOPK_EXACT_TRAP 1
|
||||
#endif
|
||||
|
||||
// ============================================================================
|
||||
// DIAGNOSTIC BUILDS ONLY -- never set in a release or serving build.
|
||||
//
|
||||
// Three compile-time defines exist so the reference host can run the two large-path
|
||||
// diagnostic cells. Every one defaults to 0 and every use
|
||||
// sits behind `#if`, so the default build's device code is identical to the
|
||||
// tree without them (offline, real TU, sm_86, against an earlier commit: SASS identical,
|
||||
// cubin identical after masking nvcc's per-compile internal-linkage TU hash,
|
||||
// which differs even between two compiles of the SAME source -- see the
|
||||
// diagnostic-build notes). A build carrying any of them prints a
|
||||
// `[merlin] persistent_topk DIAGNOSTIC BUILD` line on its first launch
|
||||
// (topk.cu), which the smoke's startup-warning scan must reject.
|
||||
//
|
||||
// MERLIN_TOPK_DIAG_DELAY=<cycles>
|
||||
// In `radix_topk`'s round loop, every NON-LEADER warp (warp != 0) of
|
||||
// every NON-LEADER CTA (cta_in_group != 0) spins <cycles> clock64 ticks
|
||||
// before merging its shared bins into the group's global histogram.
|
||||
// Warp 0 holds thread 0, the arrival publisher, so it is never delayed:
|
||||
// the delay opens exactly the window the histogram-publication barrier closes -- a
|
||||
// publisher that announces arrival before its own CTA's other warps have
|
||||
// produced. CTA 0 is never delayed, so it reads the histogram as soon
|
||||
// as the arrival count completes.
|
||||
// MERLIN_TOPK_DIAG_PREREPAIR=1
|
||||
// Compile out the histogram-publication barrier. This is the
|
||||
// PRE-REPAIR control for the delayed-publication stress: with a delay it
|
||||
// is expected to return wrong or faulted rows; the repaired build with
|
||||
// the same delay must not. Only meaningful with MERLIN_TOPK_DIAG_DELAY.
|
||||
// Do NOT run it with MERLIN_TOPK_DET_LARGE=0: the legacy tail bounds no
|
||||
// emission slot, so a disagreeing pivot writes out of bounds there.
|
||||
// MERLIN_TOPK_DIAG_FORCE_FAULT=1 (+ MERLIN_TOPK_DIAG_FORCE_FAULT_CTA, 1)
|
||||
// Force the CTA-local X3 predicate in that CTA on every deterministic
|
||||
// large row, exactly as MERLIN_TOPK_LARGE_FAULT_INJECT does -- but WITHOUT
|
||||
// suppressing `__trap()`. The trap-ON fault arm: the process's CUDA
|
||||
// context dies. Run only in a disposable subprocess.
|
||||
// ============================================================================
|
||||
#ifndef MERLIN_TOPK_DIAG_DELAY
|
||||
#define MERLIN_TOPK_DIAG_DELAY 0
|
||||
#endif
|
||||
#ifndef MERLIN_TOPK_DIAG_PREREPAIR
|
||||
#define MERLIN_TOPK_DIAG_PREREPAIR 0
|
||||
#endif
|
||||
#ifndef MERLIN_TOPK_DIAG_FORCE_FAULT
|
||||
#define MERLIN_TOPK_DIAG_FORCE_FAULT 0
|
||||
#endif
|
||||
#ifndef MERLIN_TOPK_DIAG_FORCE_FAULT_CTA
|
||||
#define MERLIN_TOPK_DIAG_FORCE_FAULT_CTA 1
|
||||
#endif
|
||||
#define MERLIN_TOPK_DIAG_ANY \
|
||||
((MERLIN_TOPK_DIAG_DELAY) != 0 || (MERLIN_TOPK_DIAG_PREREPAIR) != 0 || \
|
||||
(MERLIN_TOPK_DIAG_FORCE_FAULT) != 0)
|
||||
#if MERLIN_TOPK_DIAG_FORCE_FAULT && !MERLIN_TOPK_EXACT_TRAP
|
||||
#error "MERLIN_TOPK_DIAG_FORCE_FAULT is the trap-ON cell; it needs MERLIN_TOPK_EXACT_TRAP=1"
|
||||
#endif
|
||||
#if MERLIN_TOPK_DIAG_FORCE_FAULT && (MERLIN_TOPK_DIAG_DELAY || MERLIN_TOPK_DIAG_PREREPAIR)
|
||||
#error "build the trap-ON and the delayed-publication diagnostics separately"
|
||||
#endif
|
||||
|
||||
#if MERLIN_TOPK_DIAG_DELAY
|
||||
// Busy-wait on the SM clock. `volatile` sink so the loop is not elided.
|
||||
__device__ __noinline__ void merlin_topk_diag_spin(long long cycles) {
|
||||
const long long start = clock64();
|
||||
volatile long long now = start;
|
||||
while (now - start < cycles) {
|
||||
now = clock64();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
// Exact, bounded-memory re-selection for a single medium-path row.
|
||||
//
|
||||
// `scores` is already offset to the row. Every radix round rescans the whole
|
||||
@@ -1621,6 +1689,13 @@ __device__ __noinline__ void radix_emit_deterministic(
|
||||
static_cast<int>(cta_in_group) == fault_inject_cta) {
|
||||
my_keep_checked = local_eq + 1u;
|
||||
}
|
||||
#if MERLIN_TOPK_DIAG_FORCE_FAULT
|
||||
// DIAGNOSTIC BUILD ONLY: the same CTA-local predicate, forced at compile
|
||||
// time, so the trap below is NOT suppressed (fault_inject_cta stays -1).
|
||||
if (cta_in_group == static_cast<uint32_t>(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA)) {
|
||||
my_keep_checked = local_eq + 1u;
|
||||
}
|
||||
#endif
|
||||
if (my_keep_checked > local_eq) fault = 1u;
|
||||
if (gt_base + eq_keep_base + local_gt + my_keep >
|
||||
static_cast<uint32_t>(TopK)) {
|
||||
@@ -1954,6 +2029,14 @@ __device__ void radix_topk(const float* __restrict__ row_input,
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
#if MERLIN_TOPK_DIAG_DELAY
|
||||
// DIAGNOSTIC BUILD ONLY (see the define block): hold back the producers
|
||||
// that are NOT the arrival publisher, in every CTA that is not the leader.
|
||||
if (cta_in_group != 0u && (tx >> 5) != 0u) {
|
||||
merlin_topk_diag_spin(static_cast<long long>(MERLIN_TOPK_DIAG_DELAY));
|
||||
}
|
||||
#endif
|
||||
|
||||
for (uint32_t i = tx; i < RADIX; i += kThreadsPerBlock) {
|
||||
if (local_histogram[i] > 0) {
|
||||
atomicAdd(¤t_hist[i], local_histogram[i]);
|
||||
@@ -1976,7 +2059,9 @@ __device__ void radix_topk(const float* __restrict__ row_input,
|
||||
// nothing here -- and reads a half-written histogram, which changes the
|
||||
// pivot. The barrier must be between the producing stores and the leader's
|
||||
// publication; program order in one thread is not cross-thread ordering.
|
||||
#if !MERLIN_TOPK_DIAG_PREREPAIR
|
||||
__syncthreads();
|
||||
#endif
|
||||
|
||||
if (tx == 0) {
|
||||
red_release(&state->arrival_counter, 1);
|
||||
|
||||
@@ -202,6 +202,28 @@ void report_tie_policy_once() {
|
||||
(void)reported;
|
||||
}
|
||||
|
||||
#if MERLIN_TOPK_DIAG_ANY
|
||||
// DIAGNOSTIC BUILD ONLY (persistent_topk.cuh, "DIAGNOSTIC BUILDS ONLY"). Say
|
||||
// so once per process, on stderr, before the first launch: the diagnostic-build tests
|
||||
// key on this line to prove which build they ran against, and the smoke's
|
||||
// startup-warning scan must reject any process that prints it.
|
||||
void report_diag_build_once() {
|
||||
static const bool reported = [] {
|
||||
std::fprintf(stderr,
|
||||
"[merlin] persistent_topk DIAGNOSTIC BUILD: "
|
||||
"MERLIN_TOPK_DIAG_DELAY=%lld MERLIN_TOPK_DIAG_PREREPAIR=%d "
|
||||
"MERLIN_TOPK_DIAG_FORCE_FAULT=%d "
|
||||
"MERLIN_TOPK_DIAG_FORCE_FAULT_CTA=%d -- NOT FOR SERVING\n",
|
||||
static_cast<long long>(MERLIN_TOPK_DIAG_DELAY),
|
||||
static_cast<int>(MERLIN_TOPK_DIAG_PREREPAIR),
|
||||
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT),
|
||||
static_cast<int>(MERLIN_TOPK_DIAG_FORCE_FAULT_CTA));
|
||||
return true;
|
||||
}();
|
||||
(void)reported;
|
||||
}
|
||||
#endif
|
||||
|
||||
// Process-lifetime device counters, one block per device. The kernel only
|
||||
// ever atomicAdds into them, so the launch path needs no host sync and the
|
||||
// values accumulate across cudagraph replays.
|
||||
@@ -568,6 +590,9 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
params.tie_recent = tie_recent_enabled();
|
||||
params.exact_decode = exact_decode_enabled();
|
||||
report_tie_policy_once();
|
||||
#if MERLIN_TOPK_DIAG_ANY
|
||||
report_diag_build_once();
|
||||
#endif
|
||||
// Allocate only off the capture path; in the ordinary engine lifecycle the
|
||||
// eager warmup lands here first, long before capture. The explicit,
|
||||
// quiescent-phase entry point is the reader op.
|
||||
|
||||
Reference in New Issue
Block a user