[TopK] Determinism toggles for greedy repeatability

- MERLIN_FULL_K: deterministic Marlin MoE split-K (after unmerged upstream #46639).
- MERLIN_QSA_SORT: canonical order for the QSA selected blocks.
- MERLIN_TIE_DET: deterministic tie-break at the persistent_topk cutoff.
- MERLIN_TIE_RECENT: cutoff ties keep the highest indices.
- Decode candidate truncation is detected; one tie direction across the
  medium path; the kernel switches are registered.

Co-Authored-By: Hinata <hinata@idihore.id>
Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
2026-10-09 19:56:00 +07:00
co-authored by Hinata
parent 512ca2fc0b
commit 2918e3fda1
10 changed files with 753 additions and 119 deletions
@@ -20,7 +20,8 @@
const int32_t *__restrict__ num_tokens_past_padded_ptr, \
const float *__restrict__ topk_weights_ptr, int top_k, \
bool mul_topk_weights, int prob_m, int prob_n, int prob_k, int *locks, \
bool has_bias, bool use_atomic_add, bool use_fp32_reduce
bool has_bias, bool use_atomic_add, bool use_fp32_reduce, \
bool use_full_k
namespace MARLIN_NAMESPACE_NAME {
template <const vllm::ScalarTypeId a_type_id, // A ScalarType id
@@ -74,7 +74,8 @@ __global__ void Marlin(
int prob_k, // reduction dimension k
int* locks, // extra global storage for barrier synchronization
bool use_atomic_add, // whether to use atomic add to reduce
bool use_fp32_reduce // whether to use fp32 global reduce
bool use_fp32_reduce, // whether to use fp32 global reduce
bool use_full_k // whether to reduce over full K per block
) {}
} // namespace MARLIN_NAMESPACE_NAME
@@ -250,7 +251,8 @@ __global__ void Marlin(
int* locks, // extra global storage for barrier synchronization
bool has_bias,
bool use_atomic_add, // whether to use atomic add to reduce
bool use_fp32_reduce // whether to use fp32 global reduce
bool use_fp32_reduce, // whether to use fp32 global reduce
bool use_full_k // whether to reduce over full K per block
) {
// Each threadblock processes one "stripe" of the B matrix with (roughly) the
// same size, which might involve multiple column "slices" (of width 16 *
@@ -370,11 +372,13 @@ __global__ void Marlin(
// see https://github.com/vllm-project/vllm/pull/24722 for more details
if (global_mn_tiles > gridDim.x) {
part2_mn_tiles = global_mn_tiles % gridDim.x;
if (part2_mn_tiles * 3 <= gridDim.x) part2_mn_tiles += gridDim.x;
if (!use_full_k && part2_mn_tiles * 3 <= gridDim.x)
part2_mn_tiles += gridDim.x;
part1_mn_iters = (global_mn_tiles - part2_mn_tiles) / gridDim.x;
}
int iters = div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
int iters =
use_full_k ? k_tiles : div_ceil(k_tiles * part2_mn_tiles, gridDim.x);
if constexpr (group_blocks != -1) {
if (group_blocks >= thread_k_blocks) {
@@ -33,6 +33,35 @@
#include <torch/headeronly/util/Exception.h>
#include "libtorch_stable/torch_utils.h"
#include <cstdlib>
namespace {
// -------------------------------------------------------------------------
// MERLIN_FULL_K: the CUDA half of the unmerged upstream PR #46639
// (`use_full_k`), which makes the Marlin MoE split-K reduction order
// independent of how many blocks the launch was split across.
//
// Our own toggle rather than upstream's VLLM_BATCH_INVARIANT, because that
// one aborts engine startup on GDN models (upstream #42960) and so can never
// reach this kernel on Flash-Next.
//
// Resolved ONCE per process, not per launch, so it is a launch constant in
// exactly the sense topk.cu's exact_medium_enabled() is: a cudagraph bakes in
// the value that was live at capture time and every replay agrees with it.
// Default OFF -- enabling it by default is a separate release decision.
//
// This std::getenv is the SINGLE source of truth. vllm/envs.py carries a raw
// passthrough entry for validate_environ and the compile-factor hash only;
// nothing in Python may parse this name a second time.
// -------------------------------------------------------------------------
bool merlin_full_k() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_FULL_K");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
}();
return enabled;
}
} // namespace
#define STATIC_ASSERT_SCALAR_TYPE_VALID(scalar_t) \
static_assert(std::is_same<scalar_t, half>::value || \
@@ -183,7 +212,8 @@ exec_config_t determine_exec_config(
const vllm::ScalarType& c_type, const vllm::ScalarType& s_type, int prob_m,
int prob_n, int prob_k, int num_experts, int top_k, int thread_m_blocks,
bool m_block_size_8, int num_bits, int group_size, bool has_zp,
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms) {
bool is_zp_float, bool is_a_8bit, int stages, int max_shared_mem, int sms,
bool use_full_k) {
exec_config_t exec_cfg = exec_config_t{1, thread_config_t{-1, -1, -1}};
thread_config_t* thread_configs = thread_m_blocks > 1
? large_batch_thread_configs
@@ -228,7 +258,8 @@ exec_config_t determine_exec_config(
else
allow_count = max(min(allow_count, 2), 1);
if (prob_n / th_config.thread_n * prob_m * top_k * 4 < sms * allow_count) {
if (!use_full_k &&
prob_n / th_config.thread_n * prob_m * top_k * 4 < sms * allow_count) {
allow_count =
max(prob_n / th_config.thread_n * prob_m * top_k * 4 / sms, 1);
}
@@ -254,6 +285,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
int group_size, int dev, cudaStream_t stream, int thread_k,
int thread_n, int sms, int blocks_per_sm, bool use_atomic_add,
bool use_fp32_reduce, bool is_zp_float) {
const bool use_full_k = merlin_full_k();
int thread_m_blocks = div_ceil(moe_block_size, 16);
bool m_block_size_8 = moe_block_size == 8;
bool is_a_8bit = a_type.size_bits() == 8;
@@ -331,7 +363,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
exec_cfg = determine_exec_config(
a_type, b_type, c_type, s_type, prob_m, prob_n, prob_k, num_experts,
top_k, thread_m_blocks, m_block_size_8, num_bits, group_size, has_zp,
is_zp_float, is_a_8bit, stages, max_shared_mem, sms);
is_zp_float, is_a_8bit, stages, max_shared_mem, sms, use_full_k);
thread_tfg = exec_cfg.tb_cfg;
}
@@ -383,7 +415,7 @@ void marlin_mm(const void* A, const void* B, void* C, void* C_tmp, void* b_bias,
A_ptr, B_ptr, C_ptr, C_tmp_ptr, bias_ptr, a_s_ptr, b_s_ptr, g_s_ptr, zp_ptr,
sorted_token_ids_ptr, expert_ids_ptr, num_tokens_past_padded_ptr,
topk_weights_ptr, top_k, mul_topk_weights, prob_m, prob_n, prob_k, locks,
has_bias, use_atomic_add, use_fp32_reduce);
has_bias, use_atomic_add, use_fp32_reduce, use_full_k);
// clang-format on
}
+4
View File
@@ -485,6 +485,10 @@ void persistent_topk(const torch::stable::Tensor& logits,
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
int64_t sync);
// The slot count of THIS build's counter array, so the Python
// reader never has to guess it from an environment variable.
int64_t persistent_topk_exact_counters_slots();
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
void cooperative_topk(const torch::stable::Tensor& logits,
const torch::stable::Tensor& lengths,
+333 -59
View File
@@ -54,6 +54,19 @@ __device__ __forceinline__ auto convert_to_uint32_v2(float x) -> uint32_t {
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
// THE definition of the effective tie direction, called by
// both the host (to report it) and the device (to obey it), so the reported
// policy and the executed policy cannot drift.
//
// `MERLIN_TIE_DET=0` is the shipped default and leaves the buffered paths on
// their original racy atomicAdd rule, which has no direction; the medium-path exact
// restart is deterministic whatever happens, so it keeps the most-recent-wins
// direction the medium-path restart shipped and was measured with.
__host__ __device__ __forceinline__ bool effective_tie_recent(bool det_ties,
bool tie_recent) {
return det_ties ? tie_recent : true;
}
__device__ __forceinline__ auto convert_to_uint8(float x) -> uint8_t {
__half h = __float2half_rn(x);
uint16_t bits = __half_as_ushort(h);
@@ -147,10 +160,27 @@ struct PersistentTopKParams {
// nullptr; the kernel only ever atomicAdds into it, never reads back.
bool exact_medium;
unsigned long long* exact_counters;
// MERLIN_TIE_DET: deterministic tie-break at the k-th cutoff.
// Resolved once on the host, like exact_medium above, so it is a launch
// constant and is baked into a cudagraph at capture.
bool det_ties;
// MERLIN_TIE_RECENT: ranks ties by DESCENDING element index, so
// the surviving blocks are the most recent rather than the oldest. Both
// rules are deterministic; they differ in which tied blocks survive, and
// measurably in draft acceptance at depth. Only read when det_ties is set.
bool tie_recent;
// MERLIN_TOPK_EXACT_DECODE: on an overflowing
// DECODE row, take the same exact restart the medium path takes instead of
// ranking a truncated candidate set. Detection and the [10] counter are
// unconditional; only the repair is gated. Default OFF -- the repair costs
// four full-row rescans and the overflow incidence on real traffic has
// never been measured, which is what the counter is for.
bool exact_decode;
// MERLIN_TOPK_DET_LARGE, ANDed on the host with the capacity gate
// 2 * ctas_per_group <= RADIX (the per-CTA counts are staged in one 256-slot
// RadixRowState histogram row). Resolved once per process, so it is a
// launch constant and a cudagraph captures one value.
// 4 * ctas_per_group <= RADIX (the per-CTA counts and the two fault-verdict
// regions are staged in one 256-slot RadixRowState histogram row). Resolved
// once per process, so it is a launch constant and a cudagraph captures one
// value.
bool det_large;
// TEST ONLY. -1 (the default) disables it; any
// other value names the `cta_in_group` whose CTA-LOCAL fault predicate is
@@ -177,10 +207,17 @@ __device__ __forceinline__ uint32_t decode_bin(float x) {
return key >> 5;
}
// Returns TRUE when this row overflowed its candidate buffer AND the caller
// asked for the exact repair, in which case nothing has been emitted that the
// restart will not overwrite and the caller must run it.
// The restart itself lives with the medium path further down and is called
// from the dispatch site, so this function needs no forward declaration of it
// -- some host compilers reject one for a `__device__` template.
template <int TopK>
__device__ __noinline__ void histogram_2048_topk(
__device__ __noinline__ bool histogram_2048_topk(
const float* __restrict__ logits, int32_t* __restrict__ output_indices,
int32_t seq_len) {
int32_t seq_len, bool det_ties, bool tie_recent,
unsigned long long* __restrict__ counters, bool exact_decode) {
extern __shared__ int decode_smem[];
const int tx = threadIdx.x;
const int lane = tx & 31;
@@ -193,7 +230,18 @@ __device__ __noinline__ void histogram_2048_topk(
constexpr int MAX_ITEMS_PER_THREAD =
(HIST2048_THRESHOLD + kThreadsPerBlock - 1) / kThreadsPerBlock;
enum : int { sTHR = 0, sOUT = 1, sREF = 2, sFIN = 3, sBUF0 = 4, sBUF1 = 5 };
// sOVF claims one of the two scalars that were already
// being zeroed by the `tx < 8` loop below and never used, so the decode
// shared-memory budget is unchanged.
enum : int {
sTHR = 0,
sOUT = 1,
sREF = 2,
sFIN = 3,
sBUF0 = 4,
sBUF1 = 5,
sOVF = 6
};
// ---- Initialize scalars (prevents stale data from prior rows) ----
if (tx < 8) {
@@ -322,7 +370,7 @@ __device__ __noinline__ void histogram_2048_topk(
__syncthreads();
int remaining_k = TopK - decode_smem[SBASE + sOUT];
if (remaining_k <= 0) return;
if (remaining_k <= 0) return false;
// If all buffered elements fit, output them all (common for short seqs)
const int raw_buf0 = decode_smem[SBASE + sBUF0];
@@ -333,7 +381,7 @@ __device__ __noinline__ void histogram_2048_topk(
output_indices[base + i] = bufs[0][i];
}
__syncthreads();
return;
return false;
}
// ---- Phase 3: Deferred refinement (rare path) ----
@@ -372,6 +420,42 @@ __device__ __noinline__ void histogram_2048_topk(
const int dst = src ^ 1;
const int raw_buf = decode_smem[SBASE + sBUF0 + src];
// ---- DECODE TRUNCATION, DETECTED ------------------
//
// `DBUF` is 3,708 and the Phase-2 append guard drops everything past it,
// but the atomicAdd that hands out slots sits OUTSIDE that guard -- so
// `raw_buf` keeps counting and a post-hoc test here is exact, exactly as
// the medium path's M2 is. Dropped candidates are invisible to
// refinement AND to the deterministic tie ranking further down: ranking
// orders the survivors, it cannot recall a winner that was never
// buffered. The failing class: an 8,192-element row inside one coarse bin
// with one element larger at FP32 but in the same half-precision bin; a
// legal append order drops that winner and the row is silently wrong.
//
// The historical zero-overflow measurements cover a workload, not a
// capacity. Until this counter has been read on the real traffic, the
// determinism claim for the decode path is conditional on it reading 0.
if (__builtin_expect(raw_buf > DBUF, 0)) {
if (tx == 0 && decode_smem[SBASE + sOVF] == 0) {
decode_smem[SBASE + sOVF] = 1; // once per row, not once per pass
if (counters != nullptr) atomicAdd(&counters[kCtrDecodeOverflow], 1ull);
}
if (exact_decode) {
// Hand the row back for the same repair the medium path takes:
// throw the buffered state away and re-select from scratch with a
// full-row radix rescan. The restart writes all TopK slots (or
// fills the row with -1 and faults), so Phase 2's `> threshold`
// emissions are overwritten rather than double-counted. `raw_buf`
// is block-uniform and `exact_decode` is a launch constant, so this
// return is taken by the whole block -- which is what makes it safe
// to leave a function that still owns the shared-memory window.
__syncthreads();
return true;
}
__syncthreads();
}
const int num_buffered = (raw_buf < DBUF) ? raw_buf : DBUF;
compute_suffix_sum();
@@ -385,6 +469,9 @@ __device__ __noinline__ void histogram_2048_topk(
__syncthreads();
const int ref_thr = decode_smem[SBASE + sREF];
// Read before any thread decrements sFIN, so every thread agrees on
// how many of the tied candidates survive the cutoff.
const int fin_total = decode_smem[SBASE + sFIN];
remaining_k -= refine[0][ref_thr + 1];
const int bit_offset = 24 - pass * 8;
@@ -416,8 +503,35 @@ __device__ __noinline__ void histogram_2048_topk(
output_indices[pos] = idx;
} else if (bin == ref_thr) {
if (pass == 3) {
const int slot = atomicAdd(&decode_smem[SBASE + sFIN], -1);
if (slot > 0) output_indices[TopK - slot] = idx;
if (det_ties) {
// Every candidate that reaches here has an identical 32-bit value,
// and the racy branch below hands out slots by atomicAdd -- first
// warp to arrive wins, so WHICH tied block survives the cutoff is
// a race. Rank by element index instead and place at a position
// derived from that rank, so the surviving set is arrival-order
// independent. No atomics on this path.
//
// O(t^2) over the exactly-tied set t. Exact 32-bit ties are rare
// (19 occurrences across 12 measured requests, one element each), so
// t is small in practice -- but that is measurement at one shape,
// not a bound.
int rank = 0;
for (int j = 0; j < num_buffered; ++j) {
const int jdx = bufs[src][j];
const bool ahead = tie_recent ? (jdx > idx) : (jdx < idx);
if (!ahead) continue;
const uint32_t jf = convert_to_uint32_v2(logits[jdx]);
if (((jf >> bit_offset) & 0xFF) == static_cast<uint32_t>(ref_thr)) {
++rank;
}
}
if (rank < fin_total) {
output_indices[TopK - fin_total + rank] = idx;
}
} else {
const int slot = atomicAdd(&decode_smem[SBASE + sFIN], -1);
if (slot > 0) output_indices[TopK - slot] = idx;
}
} else {
const int bp = atomicAdd(&decode_smem[SBASE + sBUF0 + dst], 1);
if (__builtin_expect(bp < DBUF, 1)) {
@@ -430,6 +544,7 @@ __device__ __noinline__ void histogram_2048_topk(
}
__syncthreads();
}
return false;
}
// ============================================================================
@@ -457,10 +572,28 @@ __device__ __noinline__ void histogram_2048_topk(
// selected keys in output order, so that would re-introduce exactly the
// nondeterminism MERLIN_QSA_SORT / MERLIN_TIE_DET / MERLIN_TIE_RECENT exist
// to remove. Ours emits in **ascending index order** via a block scan (no
// atomics anywhere on the placement path) and cuts ties at the k-th value by
// **most-recent wins** (the highest element indices are kept), matching the
// rule measured on the reference host (+3.4 % tokens/step at 12K over
// oldest-wins).
// atomics anywhere on the placement path).
//
// TIE DIRECTION IS ONE POLICY, DECIDED CENTRALLY. The medium-path restart
// hard-coded most-recent-wins here while the buffered decode/medium paths took
// their direction from `tie_recent`, so a medium row that grew from 4,096 to
// 4,097 candidates silently changed tie direction without the configuration
// changing. The direction is now a parameter and the ONE place that resolves
// it is `effective_tie_recent()` in topk.cu:
//
// det_ties ? tie_recent : true
//
// i.e. `MERLIN_TIE_DET=1 MERLIN_TIE_RECENT=0` is oldest-wins on the whole
// medium path (buffered AND restart), and with `MERLIN_TIE_DET=0` the restart
// keeps its shipped most-recent-wins default (the direction that was
// measured at +3.4 % tokens/step at 12K over oldest-wins).
//
// The deterministic large path takes the SAME value.
// `radix_emit_deterministic` receives `effective_tie_recent(det_ties,
// tie_recent)` from the dispatch site and mirrors both halves of its cut --
// the cross-CTA walk and the within-chunk window -- so all five paths
// (decode-buffered, decode-restart, medium-buffered, medium-restart, large)
// obey one decision and the host's policy line prints it for all five.
// ============================================================================
// Abort the kernel when the exact fallback cannot complete, instead of
@@ -483,6 +616,23 @@ __device__ __noinline__ void histogram_2048_topk(
// rows and wraps after 66,053, and a delta taken across that boundary is a
// wrong number that looks like a measurement. `atomicAdd(unsigned long long*)`
// is native on sm_86, so the width costs nothing on the increment path.
//
// SLOT MAP (the tie-determinism + large-path integration, 11 slots). The tie change numbered its decode
// counter AROUND the large-path block rather than after the medium-path one, so the merged layout is
// the union of the two branches with nothing renumbered and no external probe
// silently re-reading a row count as a switch -- the failure a merged layout
// must avoid.
//
// [0] medium rows [1] medium overflow [2] medium exact fallback
// [3] medium exact fault
// [4]-[7] large path (rows_large, ties_cut, ties_dropped, faults_large)
// [8] MERLIN_TOPK_EXACT_MEDIUM [9] MERLIN_TOPK_DET_LARGE
// [10] decode overflow
//
// Slots 8-9 are written by the HOST after the device copy; the device never
// touches them, and they are slots INSIDE the array, not two elements appended
// after it. A stale five- or ten-element reader buffer is rejected outright by
// the length check in the reader op, which is the fail-loud direction.
enum TopKExactCounter : int {
kCtrMediumRows = 0, // denominator: medium-path rows entered
kCtrMediumOverflow = 1, // rows where a 4,096-slot guard would drop
@@ -497,7 +647,11 @@ enum TopKExactCounter : int {
kCtrLargeTiesCut = 5, // rows where the pivot-equal group was cut
kCtrLargeTiesDropped = 6, // pivot-equal elements discarded by those cuts
kCtrLargeFault = 7, // large-path invariant violation
kNumTopKExactCounters = 8,
kSlotSwitchMedium = 8, // host-written flag, NOT a device counter
kSlotSwitchLarge = 9, // host-written flag, NOT a device counter
kCtrDecodeOverflow = 10, // decode rows whose buffered-candidate count
// exceeded DBUF before deterministic ranking
kNumTopKExactCounters = 11, // total slots: device buffer AND reader output
};
// Exact, bounded-memory re-selection for a single medium-path row.
@@ -506,10 +660,17 @@ enum TopKExactCounter : int {
// row, so correctness does not depend on how many values share a coarse bin.
// Entered only after an overflow has been detected, which is 0.1-0.2 % of
// medium rows at 38K-51K prompt tokens.
//
// `tie_recent` is the EFFECTIVE tie direction for this launch, resolved once
// on the host (see the header comment above): true keeps the highest element
// indices among the pivot-equal group, false keeps the lowest. It is a launch
// constant and block-uniform, so every walk and the emission predicate below
// agree on it without a barrier.
template <int TopK>
__device__ __noinline__ void medium_exact_restart(
const float* __restrict__ scores, int32_t* __restrict__ output,
uint32_t length, unsigned long long* __restrict__ counters) {
uint32_t length, unsigned long long* __restrict__ counters,
bool tie_recent) {
static_assert(kThreadsPerBlock >= RADIX,
"the single-thread bin walk assumes RADIX <= blockDim.x");
@@ -521,7 +682,8 @@ __device__ __noinline__ void medium_exact_restart(
uint32_t prefix; // ordered-uint radix prefix, grows 8 bits per round
uint32_t remaining; // how many of the pivot-equal group must be kept
uint32_t tie_total; // how many elements have key == pivot in total
uint32_t tie_cut; // lowest element index kept among the pivot-equal
uint32_t tie_cut; // pivot-equal cut: lowest kept index under
// most-recent-wins, highest kept under oldest-wins
uint32_t fault; // set by the walks; block-wide, read after a barrier
};
@@ -541,7 +703,11 @@ __device__ __noinline__ void medium_exact_restart(
smem->prefix = 0;
smem->remaining = static_cast<uint32_t>(TopK);
smem->tie_total = 0;
smem->tie_cut = 0;
// The cut is a bound on the kept index range, so its identity element is
// direction-dependent. When tie_total == equal_count the walks below are
// skipped entirely and this value is what the emission predicate reads:
// `idx >= 0` keeps the whole group, and so does `idx <= 0xFFFFFFFF`.
smem->tie_cut = tie_recent ? 0u : 0xFFFFFFFFu;
// The index-space tie walk below is two 8-bit rounds, so it resolves
// indices below 65,536. The medium path serves seq_len <= 32,768, but
// fail loudly rather than silently mis-cut if that ever changes.
@@ -626,11 +792,18 @@ __device__ __noinline__ void medium_exact_restart(
// the end, but wasted work on the fault path and harder to debug. The
// condition is block-uniform (shared, read after the barrier above).
if (smem->fault == 0u) {
// ---- tie cut at the k-th value: most-recent wins --------------------
// ---- tie cut at the k-th value --------------------------------------
//
// Keep the `equal_count` HIGHEST element indices among the pivot-equal
// group. Finding the cut index turns the tie rule into a per-element
// predicate, which is what lets the emission below be atomic-free.
// group when `tie_recent`, the `equal_count` LOWEST when not. Finding the
// cut index turns the tie rule into a per-element predicate, which is what
// lets the emission below be atomic-free.
//
// The two index walks below are the only code that has
// to know the direction; they differ from each other only in which end of
// the index-bin array they start from, so `count_above` means "population
// on the far side of the walk" in both directions and the crossing test is
// unchanged.
//
// Exact 32-bit ties at the cutoff are rare (19 occurrences across
// 12 requests, one element each), so the common case is tie_total ==
@@ -651,9 +824,12 @@ __device__ __noinline__ void medium_exact_restart(
}
__syncthreads();
if (tx == 0) {
const int first_bin = tie_recent ? RADIX - 1 : 0;
const int bin_step = tie_recent ? -1 : 1;
uint32_t count_above = 0;
bool found = false;
for (int bin = RADIX - 1; bin >= 0; --bin) {
for (int n = 0; n < RADIX; ++n) {
const int bin = first_bin + n * bin_step;
const uint32_t count = smem->histogram[bin];
if (count_above + count >= need) {
smem->tie_cut = static_cast<uint32_t>(bin); // high byte, for now
@@ -684,9 +860,12 @@ __device__ __noinline__ void medium_exact_restart(
}
__syncthreads();
if (tx == 0) {
const int first_bin = tie_recent ? RADIX - 1 : 0;
const int bin_step = tie_recent ? -1 : 1;
uint32_t count_above = 0;
bool found = false;
for (int bin = RADIX - 1; bin >= 0; --bin) {
for (int n = 0; n < RADIX; ++n) {
const int bin = first_bin + n * bin_step;
const uint32_t count = smem->histogram[bin];
if (count_above + count >= need) {
smem->tie_cut = (high_byte << 8) | static_cast<uint32_t>(bin);
@@ -711,7 +890,9 @@ __device__ __noinline__ void medium_exact_restart(
// ---- emission: ascending index order, no atomics ---------------------
//
// selected(i) = ordered(i) > pivot || (ordered(i) == pivot && i >= tie_cut)
// selected(i) = ordered(i) > pivot ||
// (ordered(i) == pivot && (tie_recent ? i >= tie_cut
// : i <= tie_cut))
//
// is a pure function of the element, so the output slot is a prefix sum
// over the row rather than an arrival rank. Tiles run in ascending base
@@ -726,8 +907,8 @@ __device__ __noinline__ void medium_exact_restart(
uint32_t flag = 0;
if (idx < length) {
const uint32_t ordered = convert_to_uint32_v2(scores[idx]);
flag =
(ordered > pivot || (ordered == pivot && idx >= tie_cut)) ? 1u : 0u;
const bool keep_tie = tie_recent ? (idx >= tie_cut) : (idx <= tie_cut);
flag = (ordered > pivot || (ordered == pivot && keep_tie)) ? 1u : 0u;
}
uint32_t rank = 0;
uint32_t total = 0;
@@ -805,7 +986,8 @@ template <int TopK>
__device__ __noinline__ void histogram_256_topk(
const float* __restrict__ logits, int* __restrict__ output_indices,
int logits_offset, int seq_len, bool exact_medium,
unsigned long long* __restrict__ counters) {
unsigned long long* __restrict__ counters, bool det_ties,
bool tie_recent) {
// No default arguments on purpose: a new call site must state whether it
// wants the exact path, rather than silently inheriting the broken one.
// All shared state lives in dynamic shared memory to avoid static
@@ -973,8 +1155,14 @@ __device__ __noinline__ void histogram_256_topk(
atomicAdd(&counters[kCtrMediumExactFallback], 1ull);
}
__syncthreads(); // no thread may clobber medium_smem before this
// The restart takes the SAME effective direction as
// the buffered branch below, so crossing the 4,096-candidate capacity
// no longer flips the tie policy. `det_ties == false` means the
// buffered branch is the racy atomicAdd rule with no direction at all,
// and the restart then keeps the medium path's shipped most-recent-wins.
medium_exact_restart<TopK>(logits + logits_offset, output_indices,
static_cast<uint32_t>(seq_len), counters);
static_cast<uint32_t>(seq_len), counters,
effective_tie_recent(det_ties, tie_recent));
return;
}
__syncthreads();
@@ -1017,6 +1205,9 @@ __device__ __noinline__ void histogram_256_topk(
}
__syncthreads();
// Read before any thread decrements shared_final_k; see the decode
// path's fin_total for the reason.
const int fin_total_m = shared_final_k;
for (int i = thread_id; i < num_buffered; i += kThreadsPerBlock) {
const int idx = buffered_indices[src_buffer][i];
const float logit_value = logits[idx + logits_offset];
@@ -1027,9 +1218,35 @@ __device__ __noinline__ void histogram_256_topk(
output_indices[output_pos] = idx;
} else if (bin == threshold_bin) {
if (pass == 3) {
const int slot = atomicAdd(&shared_final_k, -1);
if (slot > 0) {
output_indices[TopK - slot] = idx;
if (det_ties) {
// Same rule as the decode path: every candidate here is bit-equal,
// so rank by element index rather than by warp arrival order.
//
// NOTE the medium-path restart interaction: when exact_medium is on AND this row
// overflowed the candidate buffer, medium_exact_restart() has
// already returned and this branch is never reached. Since the fix,
// that path is handed THIS same `tie_recent`, so the
// medium path has one tie direction either side of the capacity
// boundary.
int rank = 0;
for (int j = 0; j < num_buffered; ++j) {
const int jdx = buffered_indices[src_buffer][j];
const bool ahead = tie_recent ? (jdx > idx) : (jdx < idx);
if (!ahead) continue;
const uint32_t jf =
convert_to_uint32_v2(logits[jdx + logits_offset]);
if (static_cast<int>((jf >> bit_offset) & 0xFF) == threshold_bin) {
++rank;
}
}
if (rank < fin_total_m) {
output_indices[TopK - fin_total_m + rank] = idx;
}
} else {
const int slot = atomicAdd(&shared_final_k, -1);
if (slot > 0) {
output_indices[TopK - slot] = idx;
}
}
} else {
const int buffer_pos =
@@ -1166,12 +1383,16 @@ __device__ __forceinline__ void wait_ge(int* ptr, int target_val,
// * every CTA reads all 2*ctas_per_group counts and derives its own base by
// an exclusive prefix over `cta_in_group`, in one fixed order, so all
// CTAs agree without further communication;
// * the tie cut keeps the HIGHEST global indices (most-recent-wins, the
// `MERLIN_TIE_RECENT` policy, measured at +3.4 % tokens/step
// over oldest-wins). Chunks are handed out in ascending index order
// * the tie cut keeps the HIGHEST global indices under most-recent-wins --
// the `MERLIN_TIE_RECENT` policy, measured at +3.4 %
// tokens/step over oldest-wins, and the shipped default -- or the LOWEST
// under oldest-wins. Chunks are handed out in ascending index order
// (`my_chunk_start = cta_in_group * chunk_size`), so "the highest
// `equal_needed` pivot-equal indices" is exactly "walk CTAs from the top
// down", which the per-CTA equal counts already answer;
// down" and the oldest-wins rule is the same walk mirrored; the per-CTA
// equal counts already answer both. The direction is the launch constant
// `effective_tie_recent(det_ties, tie_recent)`, shared with the medium
// path;
// * emission is a prefix sum over ascending element index — no atomic
// anywhere on the placement path — so the row is strictly ascending in
// index and bit-reproducible.
@@ -1198,7 +1419,7 @@ __device__ __noinline__ void radix_emit_deterministic(
uint32_t* local_histogram, uint32_t* suffix_sum, uint32_t* shared_ordered,
RadixRowState* state, uint32_t cta_in_group, uint32_t ctas_per_group,
int& barrier_phase, uint32_t radix_iter, uint32_t tx,
unsigned long long* counters, int fault_inject_cta) {
unsigned long long* counters, int fault_inject_cta, bool tie_recent) {
constexpr uint32_t kWarpsPerBlock = kThreadsPerBlock / 32u;
// Staging for the published per-CTA counts. The host gates the switch on
// 4 * ctas_per_group <= RADIX, so these regions never collide.
@@ -1216,6 +1437,8 @@ __device__ __noinline__ void radix_emit_deterministic(
// [0] gt count [1] eq count [2] gt_base [3] eq_keep_base [4] cut_rank
// [5] group fault verdict [6] this CTA's emission-bound violation
// [7] group emission-bound verdict [8] ties dropped on this row
// [9] my_keep (the pivot-equal window width; with [4] it spells the cut
// in BOTH tie directions -- see the tie_recent note above)
// ---- per-CTA counts, out of the resident chunk -------------------------
if (tx < 2u) suffix_sum[tx] = 0u;
if (tx == 6u) suffix_sum[6] = 0u;
@@ -1301,18 +1524,38 @@ __device__ __noinline__ void radix_emit_deterministic(
gt_total += g;
}
// Most-recent-wins: walk CTAs from the top down, handing each one as many
// of the `equal_needed` slots as the CTAs above it left unclaimed. keep[]
// is written over eq[] in place (eq[c] is read before keep[c] is stored).
uint32_t eq_above = 0u;
for (int c = static_cast<int>(ctas_per_group) - 1; c >= 0; --c) {
const uint32_t e = local_histogram[kEqStage + c];
const uint32_t want =
(eq_above >= equal_needed) ? 0u : (equal_needed - eq_above);
local_histogram[kEqStage + c] = (want < e) ? want : e;
eq_above += e;
// Tie-determinism + large-path integration: the walk direction IS the tie
// policy. Most-recent-wins walks CTAs from the top down, handing each one
// as many of the `equal_needed` slots as the CTAs above it left unclaimed;
// oldest-wins is the same walk mirrored, from the bottom up. Chunks are
// handed out in ascending index order (`my_chunk_start = cta_in_group *
// chunk_size`), so "the highest `equal_needed` pivot-equal indices" is
// exactly "the top CTAs first" and the lowest is exactly the reverse.
// `tie_recent` is the EFFECTIVE direction, resolved once on the host by
// `effective_tie_recent(det_ties, tie_recent)` -- the same value the
// medium restart takes and the same value the host's policy line prints,
// so the three paths cannot drift. It is a launch constant and this walk
// is single-threaded, so the branch costs nothing that matters. keep[] is
// written over eq[] in place (eq[c] is read before keep[c] is stored).
uint32_t eq_seen = 0u;
if (tie_recent) {
for (int c = static_cast<int>(ctas_per_group) - 1; c >= 0; --c) {
const uint32_t e = local_histogram[kEqStage + c];
const uint32_t want =
(eq_seen >= equal_needed) ? 0u : (equal_needed - eq_seen);
local_histogram[kEqStage + c] = (want < e) ? want : e;
eq_seen += e;
}
} else {
for (uint32_t c = 0; c < ctas_per_group; ++c) {
const uint32_t e = local_histogram[kEqStage + c];
const uint32_t want =
(eq_seen >= equal_needed) ? 0u : (equal_needed - eq_seen);
local_histogram[kEqStage + c] = (want < e) ? want : e;
eq_seen += e;
}
}
const uint32_t eq_total = eq_above;
const uint32_t eq_total = eq_seen;
uint32_t eq_keep_base = 0u;
for (uint32_t c = 0; c < cta_in_group; ++c) {
@@ -1354,9 +1597,15 @@ __device__ __noinline__ void radix_emit_deterministic(
suffix_sum[2] = gt_base;
suffix_sum[3] = eq_keep_base;
// Keep the local pivot-equal elements whose ascending rank within this
// chunk is >= cut_rank, i.e. the `my_keep` highest-indexed of them.
suffix_sum[4] = (my_keep <= local_eq) ? (local_eq - my_keep) : 0u;
// The kept pivot-equal elements of this chunk are the `my_keep` whose
// ascending rank within the chunk lies in [cut_rank, cut_rank + my_keep).
// Most-recent-wins puts that window at the TOP of the chunk
// (cut_rank = local_eq - my_keep, so the upper bound is vacuous and the
// emission predicate is byte-for-byte the rule the large path shipped); oldest-wins
// puts it at the BOTTOM (cut_rank = 0, and the upper bound is what binds).
suffix_sum[4] =
tie_recent ? ((my_keep <= local_eq) ? (local_eq - my_keep) : 0u) : 0u;
suffix_sum[9] = my_keep;
suffix_sum[8] = (eq_total > equal_needed) ? (eq_total - equal_needed) : 0u;
// Publish this CTA's verdict. Own word, so no pre-zeroing is needed.
@@ -1396,6 +1645,7 @@ __device__ __noinline__ void radix_emit_deterministic(
const uint32_t gt_base = suffix_sum[2];
const uint32_t eq_keep_base = suffix_sum[3];
const uint32_t cut_rank = suffix_sum[4];
const uint32_t keep_width = suffix_sum[9];
const uint32_t row_fault = suffix_sum[5];
// ---- emission: rank by index, no atomics -------------------------------
@@ -1459,11 +1709,20 @@ __device__ __noinline__ void radix_emit_deterministic(
const uint32_t tile_gt = warp_gt[32];
const uint32_t tile_eq = warp_eq[32];
if (f_gt || (f_eq && eq_rank >= cut_rank)) {
// Selected elements before this one, in ascending index order:
// all `> pivot` ones, plus the pivot-equal ones past the cut.
const uint32_t kept_eq_before =
if (f_gt || (f_eq && eq_rank >= cut_rank &&
eq_rank - cut_rank < keep_width)) {
// Selected elements before this one, in ascending index order: all
// `> pivot` ones, plus the pivot-equal ones already inside the cut
// window. The clamp to `keep_width` is what makes the expression work
// in BOTH directions: under most-recent-wins the window ends at the
// last pivot-equal element of the chunk, so `eq_rank - cut_rank` can
// never reach `keep_width` and the clamp is dead; under oldest-wins
// the window ends early and every pivot-equal element after it must
// stop contributing, or a later `> pivot` element is placed past the
// end of the row.
uint32_t kept_eq_before =
(eq_rank > cut_rank) ? (eq_rank - cut_rank) : 0u;
if (kept_eq_before > keep_width) kept_eq_before = keep_width;
const uint32_t slot = gt_base + eq_keep_base + gt_rank + kept_eq_before;
if (slot < static_cast<uint32_t>(TopK)) {
row_output[slot] = static_cast<int32_t>(my_chunk_start + i);
@@ -1554,7 +1813,8 @@ __device__ void radix_topk(const float* __restrict__ row_input,
RadixRowState* state, uint32_t cta_in_group,
uint32_t ctas_per_group, int& barrier_phase,
uint32_t radix_iter, uint32_t tx, bool det_large,
unsigned long long* counters, int fault_inject_cta) {
unsigned long long* counters, int fault_inject_cta,
bool tie_recent) {
const uint32_t my_chunk_end = (my_chunk_start + chunk_size < seq_len)
? my_chunk_start + chunk_size
: seq_len;
@@ -1715,7 +1975,7 @@ __device__ void radix_topk(const float* __restrict__ row_input,
row_output, ordered_pivot, shared_scalars[1], my_chunk_start,
actual_chunk_size, local_histogram, suffix_sum, shared_ordered, state,
cta_in_group, ctas_per_group, barrier_phase, radix_iter, tx, counters,
fault_inject_cta);
fault_inject_cta, tie_recent);
return;
}
@@ -1854,10 +2114,23 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
row_output[i] = (i < seq_len) ? static_cast<int32_t>(i) : -1;
}
} else if (seq_len <= static_cast<uint32_t>(HIST2048_THRESHOLD)) {
histogram_2048_topk<TopK>(row_input, row_output, seq_len);
// An overflowing decode row comes back here for
// the same exact restart the medium path takes. The return is
// block-uniform and nothing has been emitted that the restart does
// not overwrite. With MERLIN_TOPK_EXACT_DECODE off this is always
// false and the decode path is byte-identical to the medium-path restart's.
const bool decode_needs_exact = histogram_2048_topk<TopK>(
row_input, row_output, seq_len, params.det_ties,
params.tie_recent, params.exact_counters, params.exact_decode);
if (decode_needs_exact) {
medium_exact_restart<TopK>(
row_input, row_output, seq_len, params.exact_counters,
effective_tie_recent(params.det_ties, params.tie_recent));
}
} else {
histogram_256_topk<TopK>(row_input, row_output, 0, seq_len,
params.exact_medium, params.exact_counters);
params.exact_medium, params.exact_counters,
params.det_ties, params.tie_recent);
}
}
continue;
@@ -1868,7 +2141,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
row_input, row_output, seq_len, my_chunk_start, chunk_size,
local_histogram, suffix_sum, shared_scalars, shared_ordered, state,
cta_in_group, ctas_per_group, barrier_phase, radix_iter, tx,
params.det_large, params.exact_counters, params.fault_inject_cta);
params.det_large, params.exact_counters, params.fault_inject_cta,
effective_tie_recent(params.det_ties, params.tie_recent));
radix_iter++;
}
}
+164 -19
View File
@@ -30,6 +30,41 @@ namespace {
// behaviour exactly (the counters still record, so the control arm reports the
// overflow incidence it is the control for).
// ---------------------------------------------------------------------------
// -------------------------------------------------------------------------
// MERLIN_TIE_DET: deterministic tie-break at the k-th cutoff in
// persistent_topk. Resolved once per process, exactly like
// exact_medium_enabled() below, so it is a launch constant that a cudagraph
// bakes in at capture.
//
// Our own toggle rather than upstream's VLLM_BATCH_INVARIANT, which aborts
// engine startup on GDN models (upstream #42960) and so can never reach this
// kernel on Flash-Next. Default OFF -- promotion is a release decision.
//
// This std::getenv is the SINGLE source of truth; vllm/envs.py carries a raw
// passthrough for validate_environ and the compile-factor hash only.
// -------------------------------------------------------------------------
// MERLIN_TIE_RECENT -- the DIRECTION of the deterministic tie-break above.
// Measured at 12k tokens on the reference host: oldest-wins costs -7.9 % decode,
// most-recent-wins is +1.5 %. Both deterministic. Also the direction
// medium_exact_restart() already implements, so setting this alongside
// MERLIN_TIE_DET makes the medium path agree with itself on overflowing
// rows. Only read when MERLIN_TIE_DET is set. Default OFF.
bool tie_recent_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TIE_RECENT");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
}();
return enabled;
}
bool det_ties_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TIE_DET");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
}();
return enabled;
}
bool exact_medium_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
@@ -56,6 +91,23 @@ bool det_large_enabled() {
return enabled;
}
// MERLIN_TOPK_EXACT_DECODE: repair an overflowing decode
// row with the same exact restart the medium path uses. Resolved once per
// process like the others, so it is a launch constant a cudagraph bakes in.
//
// Default OFF, unlike MERLIN_TOPK_EXACT_MEDIUM. The medium path's overflow
// incidence was measured (0.1-0.2 % of medium rows at 38-51K) before that
// switch defaulted on; the decode path's has never been measured at all,
// which is precisely what the [10] counter now exists to establish. Detection
// is unconditional and free; the repair waits on a number.
bool exact_decode_enabled() {
static const bool enabled = [] {
const char* raw = std::getenv("MERLIN_TOPK_EXACT_DECODE");
return (raw != nullptr && raw[0] != '\0') ? (std::atoi(raw) != 0) : false;
}();
return enabled;
}
// ---------------------------------------------------------------------------
// MERLIN_TOPK_LARGE_FAULT_INJECT. TEST ONLY.
//
@@ -80,6 +132,50 @@ int large_fault_inject_cta() {
return cta;
}
// ---------------------------------------------------------------------------
// Say the effective tie policy out loud, once.
//
// "TIE_DET=1 TIE_RECENT=0" used to mean three different things on the three
// paths, and nothing printed the difference. One stderr line at the first
// launch of the process naming the policy the kernel will actually apply on
// EVERY path. It is a plain fprintf on the host: no CUDA call, so it is safe
// from a capturing thread, and the `static` initialiser makes it exactly once
// per process regardless of how many workers share the log.
//
// The line covers all FIVE paths. The large
// path is no longer a fixed rule: `radix_emit_deterministic` takes the same
// `effective_tie_recent()` value as the medium restart, so `large=` reports
// the real direction when MERLIN_TOPK_DET_LARGE is on and the inherited
// v2.0.1 atomic-arrival race when it is off. The flag printed here is the
// ENV value; a launch whose ctas_per_group overflows the staging row degrades
// to atomic-arrival for that launch and leaves counter [4] at 0, which is the
// separate warning below.
// ---------------------------------------------------------------------------
void report_tie_policy_once() {
static const bool reported = [] {
const bool det = det_ties_enabled();
const bool recent =
vllm::persistent::effective_tie_recent(det, tie_recent_enabled());
const char* dir = recent ? "most-recent-wins" : "oldest-wins";
std::fprintf(
stderr,
"[merlin] persistent_topk tie policy: MERLIN_TIE_DET=%d "
"MERLIN_TIE_RECENT=%d MERLIN_TOPK_EXACT_MEDIUM=%d "
"MERLIN_TOPK_EXACT_DECODE=%d MERLIN_TOPK_DET_LARGE=%d -> "
"decode-buffered=%s decode-restart=%s medium-buffered=%s "
"medium-restart=%s large=%s\n",
det ? 1 : 0, tie_recent_enabled() ? 1 : 0,
exact_medium_enabled() ? 1 : 0, exact_decode_enabled() ? 1 : 0,
det_large_enabled() ? 1 : 0, det ? dir : "atomic-arrival",
exact_decode_enabled() ? dir : "n/a (detect-only; counter [10])",
det ? dir : "atomic-arrival",
exact_medium_enabled() ? dir : "n/a (exact restart disabled)",
det_large_enabled() ? dir : "atomic-arrival (non-deterministic)");
return true;
}();
(void)reported;
}
// Process-lifetime device counters, one block per device. The kernel only
// ever atomicAdds into them, so the launch path needs no host sync and the
// values accumulate across cudagraph replays.
@@ -437,6 +533,10 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
params.fault_inject_cta);
}
}
params.det_ties = det_ties_enabled();
params.tie_recent = tie_recent_enabled();
params.exact_decode = exact_decode_enabled();
report_tie_policy_once();
// Allocate only off the capture path; in the ordinary engine lifecycle the
// eager warmup lands here first, long before capture. The explicit,
// quiescent-phase entry point is the reader op.
@@ -542,9 +642,10 @@ void persistent_topk(const torch::stable::Tensor& logits,
// device before any cudagraph capture.
//
// PROTOCOL VERSION 2 -- BREAKING. `out` is an **int64**
// CUDA tensor with at least 10 elements; int32 is REJECTED, not silently
// accepted. The device counters are 64-bit unsigned and are copied D2D
// without conversion, so the reader's element width must match the device's.
// CUDA tensor with EXACTLY kNumTopKExactCounters (11) elements; int32, and any
// other length, are REJECTED rather than silently accepted. The device counters are 64-bit unsigned
// and are copied D2D without conversion, so the reader's element width must
// match the device's.
//
// Why the break rather than a reinterpreting reader: the counters were u32
// copied into a SIGNED int32 tensor. At 65,536 columns with k=512 an
@@ -559,11 +660,20 @@ void persistent_topk(const torch::stable::Tensor& logits,
// so the instrument costs the same, and a 64-bit slot cannot be driven to the
// signed boundary by any run this kernel will ever see.
//
// The layout is the UNION of the tie-determinism and large-path slot
// maps -- the large path's ten slots plus the decode restart's [10] decode_overflow, eleven in all,
// with nothing renumbered. The two switch flags are slots INSIDE the array at
// [8] and [9], not two elements appended after it, so a probe sized to the
// old ten- or five-element layout is rejected by the length check below rather
// than being filled with the wrong meanings. Ask the build for its own answer
// with `torch.ops._C.persistent_topk_exact_counters_slots()`.
//
// Everything that reads these counters must move together: the Python reader
// `vllm/models/qwen4_exp/nvidia/ops/topk_counters.py` now allocates int64, and
// the medium-path tests allocate int64. An int32 caller gets a loud error rather
// than half a number. The OPERATOR SCHEMA is unchanged (dtype is not part of
// it), so a stale caller fails at the check, not at the binding.
// `vllm/models/qwen4_exp/nvidia/ops/topk_counters.py` allocates int64 and
// eleven slots, and the medium- and large-path tests do too. An int32 caller gets a loud
// error rather than half a number. The OPERATOR SCHEMA is unchanged (dtype
// and length are not part of it), so a stale caller fails at the check, not at
// the binding.
//
// Slots:
// [0] medium-path rows entered (denominator)
@@ -574,8 +684,11 @@ void persistent_topk(const torch::stable::Tensor& logits,
// [5] large-path rows whose pivot-equal group was cut at the k-th value
// [6] large-path pivot-equal elements discarded by those cuts
// [7] large-path invariant violations (must stay 0)
// [8] 1 if MERLIN_TOPK_EXACT_MEDIUM is on, else 0
// [9] 1 if MERLIN_TOPK_DET_LARGE is on, else 0
// [8] 1 if MERLIN_TOPK_EXACT_MEDIUM is on, else 0 (host-written)
// [9] 1 if MERLIN_TOPK_DET_LARGE is on, else 0 (host-written)
// [10] decode rows whose buffered candidates exceeded DBUF (3,708)
// before deterministic ranking -- MUST be read before any decode-path
// determinism claim
//
// COHERENCY -- stated precisely, because the earlier wording overclaimed.
// With `sync == 0` the snapshot is ordered ONLY against work already enqueued
@@ -603,7 +716,8 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
out.scalar_type() == torch::headeronly::ScalarType::Long,
"out must be int64: the exact-restart counters are 64-bit as of the "
"large-path counter protocol change, and an int32 buffer would be filled "
"from half of each counter. Allocate torch.zeros(10, dtype=torch.int64)."
"from half of each counter. Allocate "
"torch.zeros(persistent_topk_exact_counters_slots(), dtype=torch.int64)."
" (int32 was protocol version 1 and is deliberately rejected.)");
// Element count alone is not enough. A CUDA scalar expanded
// to ten elements (stride 0) satisfies numel but owns one element's storage,
@@ -614,9 +728,19 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
STD_TORCH_CHECK(out.is_contiguous(),
"out must be contiguous (expanded or strided views would be "
"written out of bounds or into the wrong elements)");
STD_TORCH_CHECK(out.numel() >= vllm::persistent::kNumTopKExactCounters + 2,
"out must hold at least ",
vllm::persistent::kNumTopKExactCounters + 2, " elements");
// EXACTLY the build's slot count, not "at least". `>=` accepted an
// 11-element buffer on the ten-slot large-path build and quietly left the extra
// element reading 0 -- a caller compiled against one layout got a plausible
// answer from a build with another, which is the failure mode the
// self-describing `..._slots` op exists to prevent. Ask the build, size the
// buffer to what it says, and a mismatch is an error rather than a wrong
// number.
STD_TORCH_CHECK(out.numel() == vllm::persistent::kNumTopKExactCounters,
"out must hold exactly ",
vllm::persistent::kNumTopKExactCounters,
" elements (this build's persistent_topk_exact_counters_"
"slots(); the switch flags are slots inside the array, not "
"appended after it), got ", out.numel());
STD_TORCH_CHECK(
reset == 0 || sync != 0,
"persistent_topk_exact_counters: reset requires sync != 0, otherwise the "
@@ -661,15 +785,20 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
}
// Static storage: the source of an async H2D copy must outlive the copy.
// Two flags: [kNumTopKExactCounters] = the medium-path switch,
// [kNumTopKExactCounters + 1] = the large-path switch. The large
// flag reports the ENV value, not the per-launch effective value: a launch
// whose ctas_per_group overflowed the staging row still reports 1 here and
// leaves counter [4] at 0, which is how that degradation is diagnosed.
// Written AFTER the device copy, which has already zeroed both slots: they
// are slots [8] and [9] INSIDE the array and the device never touches them.
// The large flag reports the ENV value, not the per-launch effective value:
// a launch whose ctas_per_group overflowed the staging row still reports 1
// here and leaves counter [4] at 0, which is how that degradation is
// diagnosed. kSlotSwitchLarge == kSlotSwitchMedium + 1, so one copy does
// both.
static_assert(vllm::persistent::kSlotSwitchLarge ==
vllm::persistent::kSlotSwitchMedium + 1,
"the two host-written flags must stay adjacent for this copy");
static const int64_t enabled[2] = {exact_medium_enabled() ? 1 : 0,
det_large_enabled() ? 1 : 0};
STD_TORCH_CHECK(
cudaMemcpyAsync(dst + vllm::persistent::kNumTopKExactCounters, enabled,
cudaMemcpyAsync(dst + vllm::persistent::kSlotSwitchMedium, enabled,
2 * sizeof(int64_t), cudaMemcpyHostToDevice,
stream) == cudaSuccess,
"exact counter flag write failed");
@@ -686,3 +815,19 @@ void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
#endif
}
// How many int32 slots THIS build's counter array carries.
//
// The Python reader used to guess from MERLIN_TOPK_COUNTER_SLOTS (default 5)
// and name the slots by that length. The decode restart makes the array 11 and the large path makes
// it 10, so a guess is now wrong on at least one branch, and a reader that
// allocates the wrong length is rejected by the length check above -- the
// instrument goes dark exactly when it starts mattering. Ask the build
// instead. No CUDA call, no allocation, safe from anywhere.
int64_t persistent_topk_exact_counters_slots() {
#ifndef USE_ROCM
return static_cast<int64_t>(vllm::persistent::kNumTopKExactCounters);
#else
return 0;
#endif
}
+3
View File
@@ -589,6 +589,7 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
// Read (and optionally reset) the medium-path exact-restart counters.
ops.def(
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
ops.def("persistent_topk_exact_counters_slots() -> int");
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
ops.def(
@@ -864,6 +865,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
ops.impl("persistent_topk_exact_counters",
TORCH_BOX(&persistent_topk_exact_counters));
ops.impl("persistent_topk_exact_counters_slots",
TORCH_BOX(&persistent_topk_exact_counters_slots));
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
ops.impl("cooperative_topk", TORCH_BOX(&cooperative_topk));
#endif
+125 -26
View File
@@ -461,32 +461,12 @@ if TYPE_CHECKING:
# an identity, it does not choose one. Ignored factor.
VLLM_E2_EXPECTED_CUDAGRAPH_MODE: str = ""
#
# --8<-- [start:fork-unaudited-knobs]
# --- TODO: knobs that are NOT in this tree at all ----------
# The tie-determinism toggles and the BSM pin live in a separate
# patch stack, not in v2.0.1 and not in this tree:
#
# MERLIN_FULL_K MERLIN_QSA_SORT MERLIN_TIE_DET MERLIN_TIE_RECENT
# MERLIN_PIN_BSM
#
# They are therefore UNAUDITED: nothing in this repository shows what they
# change, and this file will not classify code it cannot read. The rule they
# fall under is the standing one -- an os.environ (or std::getenv) knob that
# changes a compiled or captured artifact is not in the cache key unless it
# is registered here. From what is known of them (Marlin split-K, a
# persistent_topk ordering change and a cutoff tie-break) they are very
# likely capture factors and possibly AOT factors, which is exactly why a
# guess is not good enough.
#
# WHEN THE PATCHES LAND, for each name: register it below, label both
# identities from its actual read site, hash it unless exclusion is proven,
# and delete it from this block. tests/test_envs.py holds a test that fails
# if a name is registered while still listed here, so the classification
# cannot be skipped silently. That test reads ONLY the delimited block
# below/above -- scanning the whole file could never pass, because a
# registry entry necessarily contains its own name.
# ----------------------------------------------------------------------
# --8<-- [end:fork-unaudited-knobs]
# The unaudited-knob fence that stood here listed the
# tie-determinism toggles as not-yet-in-tree. They landed with the
# v2.2 kernel port: the four are registered and
# classified in the "determinism toggles" block below, and the
# one knob still unclassified sits in that block's own fence, which
# tests/test_envs.py reads as a set of identifiers.
VLLM_LOG_MODEL_INSPECTION: bool = False
VLLM_DEBUG_MFU_METRICS: bool = False
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
@@ -512,6 +492,112 @@ if TYPE_CHECKING:
VLLM_NIC_SELECTION_VARS: str = ""
VLLM_ENABLE_HPC_OPS: bool = False
MERLIN_TOPK_EXACT_COUNTERS_LOG: int = 0
#
# --- determinism toggles -----------------------------------------
# The three nondeterminism mechanisms (FULL_K, QSA_SORT, TIE_DET/TIE_RECENT).
# Every one of them is
# OFF by default: promotion is the maintainer's decision.
#
# Why these are registered at all: validate_environ() only inspects names
# beginning with VLLM_, so a MERLIN_ name never trips the unknown-variable
# warning. Registration here exists for ONE reason -- compile_factors() can
# only hash names it knows, and compile_factors() is a deny-list, so
# registering a name hashes it unless ignored_factors says otherwise. All
# four are deliberately absent from ignored_factors.
#
# CAPTURE: yes, all four. Each resolves once per process and the resolved
# value then reaches the GPU as a kernel launch argument, a launch
# configuration, or an extra op in the traced sequence -- so a cudagraph
# bakes in whatever was live at capture time. Flipping one between boots
# changes the captured graph, which is exactly what a cache key is for.
# AOT: not established for the three CUDA toggles (the Marlin and topk
# kernels are compiled into the extension at build time, not by the AOT
# compiler); established for MERLIN_QSA_SORT, which adds torch ops to the
# traced graph. Hashed conservatively throughout, the same rule
# already applied to MERLIN_TOPK_EXACT_MEDIUM.
#
# NOT A PYTHON READ SITE (FULL_K / TIE_DET / TIE_RECENT): each is resolved
# once by a std::getenv in C++ -- merlin_full_k() in
# csrc/libtorch_stable/moe/marlin_moe_wna16/ops.cu and merlin_det_ties() /
# merlin_tie_recent() in csrc/libtorch_stable/topk.cu -- and that is the
# single source of truth. The entries below are raw passthroughs so nothing
# can mistake them for a second parser; do not use them to decide
# behaviour. A raw passthrough hashes "0" and "00" differently even though
# the C++ treats them alike: over-invalidation, which is the safe direction.
#
# Marlin MoE split-K reduction order; the CUDA half of unmerged upstream
# PR #46639 (`use_full_k`).
MERLIN_FULL_K: str | None = None
# persistent_topk tie-break at the k-th cutoff, and its direction.
MERLIN_TIE_DET: str | None = None
MERLIN_TIE_RECENT: str | None = None
#
# A REAL Python read site, unlike the three above: parsed here and read
# once at import by models/qwen4_exp/nvidia/ops/qsa_indexer.py, which adds
# a torch.sort + two torch.where ops to the QSA selection path.
MERLIN_QSA_SORT: bool = False
#
# --- persistent_topk kernel switches ------------------------------
# Same classification as the three raw passthroughs above and registered
# for the same single reason: compile_factors() is a deny-list, so a name
# it does not know cannot be hashed, and a Ray deployment that propagates
# registered variables can otherwise drop an explicitly-disabled switch
# and leave a worker on the default. Each is resolved ONCE per process by
# a std::getenv in csrc/libtorch_stable/topk.cu -- that is the single
# source of truth -- and the resolved value reaches the GPU as a kernel
# launch argument, so a cudagraph bakes in whatever was live at capture.
# CAPTURE: yes, all three. AOT: not established (these kernels are built
# into the extension, not by the AOT compiler); hashed conservatively
# anyway, same rule as above. None is in ignored_factors.
#
# Raw passthroughs, so "unset" reads as None here. That is NOT the same as
# "off" for the C++: MERLIN_TOPK_EXACT_MEDIUM defaults ON. Read the getter
# in topk.cu, not this default, for the shipped behaviour.
#
# Take the exact restart on an overflowing MEDIUM row:
# MERLIN_TOPK_EXACT_MEDIUM. Declared ONCE, in the cross-change knob
# block above (same raw passthrough, same hashing); the 0.30 tree already
# carried that entry, so it is not repeated here.
# The same restart on an overflowing DECODE row.
MERLIN_TOPK_EXACT_DECODE: str | None = None
# Deterministic placement and tie membership on the LARGE path.
# Registered here ahead of its code, exactly as the four toggles above
# were, so the full tree is complete; its read site is
# det_large_enabled() in topk.cu on the large path. Setting it without that
# code does nothing, which is why it is a passthrough and not a parsed
# bool that would imply this tree can act on it.
MERLIN_TOPK_DET_LARGE: str | None = None
#
# NOT REGISTERED, deliberately: MERLIN_TOPK_EXACT_TRAP. It is a
# preprocessor define consumed by `#ifndef` in persistent_topk.cuh at
# BUILD time (-D), not a process environment variable -- nothing in the
# tree calls getenv on it. Registering it would advertise a runtime knob
# that does not exist. Its default (trap ON) is an open maintainer decision,
# not something to settle by
# quietly making it settable.
#
# --- UNAUDITED: reference-host knobs with no read site in this tree ----------
# A knob named inside the fence below is one this repository cannot
# classify, because the code that reads it is not in it. Registering one
# without first deleting it from the fence -- which means writing down its
# AOT and capture identity against its real read site -- is the exact
# failure the fork's env rule exists to prevent, and
# tests/test_envs.py::test_unaudited_host_side_knob_stays_listed_until_it_is_classified
# fails on it.
#
# The fence holds NOTHING BUT the names, one per line. Prose belongs out
# here: the gate reads the fence as a set of identifiers, and a knob name
# mentioned in passing inside it would otherwise read as a listing.
#
# The one entry, MERLIN_PIN_BSM, pins Marlin's block_size_m to 64 (Python,
# no rebuild). It is an ALTERNATIVE to the split-K toggle above, not a
# complement: on its own it fixed only the 74/102-token cells and left
# 1050 upward broken, while use_full_k fixed all of them with block_size_m
# still varying. The tie-determinism change promotes use_full_k and keeps this one off, so its
# read site was never brought across and it stays unclassified.
# --8<-- [start:unaudited-host-side-knobs]
# MERLIN_PIN_BSM
# --8<-- [end:unaudited-host-side-knobs]
def get_default_cache_root():
@@ -2546,6 +2632,19 @@ environment_variables: dict[str, Callable[[], Any]] = {
"MERLIN_TOPK_EXACT_COUNTERS_LOG": lambda: int(
os.getenv("MERLIN_TOPK_EXACT_COUNTERS_LOG", "0")
),
# Determinism toggles. See the declaration block above for the AOT /
# capture classification and for why three of these are raw passthroughs.
# All four are absent from ignored_factors on purpose, so all four are
# hashed into the compile cache key.
"MERLIN_FULL_K": lambda: os.getenv("MERLIN_FULL_K", None),
"MERLIN_TIE_DET": lambda: os.getenv("MERLIN_TIE_DET", None),
"MERLIN_TIE_RECENT": lambda: os.getenv("MERLIN_TIE_RECENT", None),
"MERLIN_QSA_SORT": lambda: bool(int(os.getenv("MERLIN_QSA_SORT", "0"))),
# persistent_topk kernel switches. Raw passthroughs
# of values C++ parses; see the declaration block above.
# MERLIN_TOPK_EXACT_MEDIUM is registered once, earlier in this dict.
"MERLIN_TOPK_EXACT_DECODE": lambda: os.getenv("MERLIN_TOPK_EXACT_DECODE", None),
"MERLIN_TOPK_DET_LARGE": lambda: os.getenv("MERLIN_TOPK_DET_LARGE", None),
}
@@ -23,6 +23,46 @@ _TOPK_WORKSPACE_BYTES = 1024 * 1024
_DECODE_BLOCK_N = 64
# ---------------------------------------------------------------------------
# MERLIN_QSA_SORT: canonicalise the ORDER of the selected block indices.
#
# persistent_topk assigns output slots by atomicAdd, so two identical requests
# get the same selected SET in a different ORDER (measured on the reference host: matching
# sorted hashes, differing raw hashes). Sparse attention then accumulates over
# those blocks in whatever order it was handed, and fp32 addition is not
# associative, so the arithmetic moves. Sorting is semantically a no-op -- the
# same blocks are attended either way -- and makes the accumulation order
# canonical.
#
# Padding (negative sentinels) is pushed to the END so the valid prefix is
# preserved: `_expand_qsa_indices_kernel` truncates by POSITION
# (`complete_blocks = min(visible_blocks, BLOCK_TOPK)`), and position is never
# compared against a score anywhere in the chain, so there is no relevance-rank
# contract for the sort to break.
#
# Branch-free and sync-free on purpose. `.any()` / `.item()` raise
# cudaErrorStreamCaptureUnsupported inside graph capture; that cost one boot
# on the reference host and is why this version has no Python-visible branch on data.
#
# Resolved once at import so it is a launch constant: a cudagraph captures the
# op sequence that was live at capture time. Default OFF -- promotion is a release
# decision, which is the maintainer's.
# ---------------------------------------------------------------------------
_QSA_SORT = envs.MERLIN_QSA_SORT
def _qsa_canonicalise(block_indices: torch.Tensor) -> None:
"""Sort selected block indices ascending, padding last. In place, shape preserved."""
if not _QSA_SORT:
return
big = torch.iinfo(block_indices.dtype).max
tmp = torch.where(
block_indices < 0, torch.full_like(block_indices, big), block_indices
)
tmp = torch.sort(tmp, dim=-1).values
block_indices.copy_(torch.where(tmp == big, torch.full_like(tmp, -1), tmp))
@triton.jit
def _qsa_mqa_paged_uniform_kernel(
q_ptr,
@@ -568,6 +608,7 @@ def _topk(
block_topk,
logits.shape[1],
)
_qsa_canonicalise(block_indices)
def qsa_select_paged_decode(
@@ -17,22 +17,40 @@ of each counter. An older build whose op still demands int32 will therefore
raise here; that is the intended signal, not a regression to paper over.
SLOT LAYOUT is build-dependent, so nothing here hard-codes five. `slot_count()`
asks the build first (a `..._slots` op, should a later build expose one) and
otherwise reads `MERLIN_TOPK_COUNTER_SLOTS` (default 5); names come from a
table keyed by that length:
asks the build first -- the tie-determinism change added the `..._slots` op this was written to look
for, so on a build that has it the env variable is never consulted -- and
otherwise reads `MERLIN_TOPK_COUNTER_SLOTS` (default 5). The op demands
EXACTLY that many elements, so a guessed length is an error, not a short read;
names come from a table keyed by that length:
5 rows overflow fallbacks faults switch
10 rows overflow fallbacks faults rows_large ties_cut ties_dropped
faults_large switch switch_large
11 the ten above, plus decode_overflow
An unknown length degrades to `slot<i>` names rather than mislabelling.
WHY 11 AND NOT 10. The tie-determinism change numbered its one new counter around the large-path block
instead of after the medium-path one, so the integration renumbers nothing: slots 4-7 are
the large-path counters, 8-9 the two switch flags, 10 the decode overflow.
On the integrated build every one of the eleven is live; on a single-branch
build the other branch's slots are structurally zero and the line still
reports them by name -- they are the slots they will hold, not a
mislabelling. `decode_overflow` is the number that has to be read before any
claim that the DECODE path is deterministic: it counts rows whose buffered
candidates exceeded the 3,708-slot capacity before the tie ranking ran, which
ranking cannot repair.
HOW THE SMOKE READS IT. Set `MERLIN_TOPK_EXACT_COUNTERS_LOG=<seconds>` on the
engine; each GPU worker then emits, at most that often and once more at
shutdown, one line into the ordinary engine log:
topk-exact-counters rank=PP0_TP1 rows=... overflow=... fallbacks=...
faults=... switch=...
faults=... switch=... [rows_large=... ... decode_overflow=...]
The first five fields are the medium-path line and keep their order on every build;
any further slot is appended after them, so an 11-slot build reads as the medium-path
line plus the four large-path fields, `switch_large` and `decode_overflow`.
Counts are cumulative per worker process, so the smoke takes the LATEST line
per rank -- `grep topk-exact-counters <engine log> | ...` keyed on the `rank=`
@@ -73,7 +91,14 @@ _LARGEDET_NAMES = (
"switch",
"switch_large",
)
_SLOT_NAMES: dict[int, tuple[str, ...]] = {5: _MEDIUM_NAMES, 10: _LARGEDET_NAMES}
# The decode-overflow counter comes after the large-path block; see the module
# docstring for why it is slot 10 and not slot 4.
_TIEDET_NAMES = _LARGEDET_NAMES + ("decode_overflow",)
_SLOT_NAMES: dict[int, tuple[str, ...]] = {
5: _MEDIUM_NAMES,
10: _LARGEDET_NAMES,
11: _TIEDET_NAMES,
}
# The medium-path line contract. Any further name is appended after these, in slot
# order, so a 10-slot build still starts with exactly the medium-path line.
@@ -92,7 +117,13 @@ def _get_op(name: str):
def slot_count() -> int:
"""How many int64 slots this build's counter array carries."""
"""How many int64 slots this build's counter array carries.
The build is asked first and the environment is only a fallback for builds
older than the slot-count op: a guessed length that does not match the extension is
rejected by the op's own length check, so the instrument would go dark
exactly when the layout changed -- which is when it matters most.
"""
query = _get_op(_OP_NAME + "_slots")
if query is not None:
return int(query())