[TopK] Exact restart of the medium path on candidate-buffer overflow
When the persistent top-k medium-path candidate buffer overflows, the row is restarted exactly instead of being truncated. Capture-safe counters with atomic publication and a coherent snapshot record every restart. Co-Authored-By: Hinata <hinata@idihore.id> Signed-off-by: MOVZX <movzx@yahoo.com>
This commit is contained in:
@@ -481,6 +481,10 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
||||
torch::stable::Tensor& workspace, int64_t k,
|
||||
int64_t max_seq_len);
|
||||
|
||||
// Medium-path exact-restart counters; see topk.cu for the slot layout.
|
||||
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||
int64_t sync);
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
void cooperative_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
|
||||
@@ -27,7 +27,9 @@ constexpr int RADIX = 256;
|
||||
// which would inflate the kernel's smem footprint and kill occupancy
|
||||
// for the decode/trivial paths).
|
||||
constexpr size_t kMediumHistBytes = 2 * (RADIX + 128) * sizeof(int); // 3072
|
||||
constexpr size_t kMediumScalarsBytes = 5 * sizeof(int); // 20
|
||||
// 6 scalars, not 5: the medium-path exact restart adds a per-row "overflow already counted" flag.
|
||||
// kMediumHeaderSize rounds to 128 B either way, so kSmemMedium is unchanged.
|
||||
constexpr size_t kMediumScalarsBytes = 6 * sizeof(int); // 24
|
||||
constexpr size_t kMediumHeaderSize =
|
||||
(kMediumHistBytes + kMediumScalarsBytes + 127) & ~size_t(127); // 3200
|
||||
constexpr int MAX_BUFFERED_ITEMS = 4096;
|
||||
@@ -138,6 +140,13 @@ struct PersistentTopKParams {
|
||||
uint32_t chunk_size; // large path: elements per CTA
|
||||
uint32_t ctas_per_group; // 1=medium, >1=large
|
||||
uint32_t max_seq_len; // max seq_len across all rows (for early CTA exit)
|
||||
// Medium-path exact top-k restart. `exact_medium` is the
|
||||
// MERLIN_TOPK_EXACT_MEDIUM switch, resolved once on the host so it is a
|
||||
// launch constant and is baked into a cudagraph at capture. `exact_counters`
|
||||
// is a process-lifetime device buffer of kNumTopKExactCounters uint32s, or
|
||||
// nullptr; the kernel only ever atomicAdds into it, never reads back.
|
||||
bool exact_medium;
|
||||
unsigned int* exact_counters;
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
@@ -411,6 +420,348 @@ __device__ __noinline__ void histogram_2048_topk(
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Medium-path exact top-k restart
|
||||
//
|
||||
// The medium path buffers the coarse threshold bin's candidates into a fixed
|
||||
// 4,096-slot shared buffer and increments the NEXT-LEVEL histogram *inside*
|
||||
// the capacity guard, so a candidate dropped on overflow is invisible to
|
||||
// refinement in both the buffer and the histogram. At 51,481 prompt tokens
|
||||
// (~12.9K candidate blocks) that makes the returned set non-exact on 88 % of
|
||||
// the rows that overflow, and which candidates survive is decided by atomicAdd
|
||||
// arrival order.
|
||||
//
|
||||
// The repair is the mechanism of vLLM PR #52149 with a different emission
|
||||
// rule: detect the overflow, then throw away everything the buffered path did
|
||||
// and re-select from scratch (`prefix = 0`, `remaining = TopK`) with a
|
||||
// full-row radix rescan. Because the restart starts from an empty prefix and
|
||||
// an undebited `remaining`, the coarse pass's winners are re-counted rather
|
||||
// than double-counted -- the "coarse handoff" defect of an earlier revision
|
||||
// cannot occur by construction.
|
||||
//
|
||||
// Where we diverge from #52149: its rescan emits by raw `atomicAdd` arrival
|
||||
// order in both the `> pivot` and `== pivot` phases, which is set-exact but
|
||||
// order- and tie-membership-nondeterministic. Sparse attention sums the
|
||||
// selected keys in output order, so that would re-introduce exactly the
|
||||
// nondeterminism MERLIN_QSA_SORT / MERLIN_TIE_DET / MERLIN_TIE_RECENT exist
|
||||
// to remove. Ours emits in **ascending index order** via a block scan (no
|
||||
// atomics anywhere on the placement path) and cuts ties at the k-th value by
|
||||
// **most-recent wins** (the highest element indices are kept), matching the
|
||||
// rule measured on the reference host (+3.4 % tokens/step at 12K over
|
||||
// oldest-wins).
|
||||
// ============================================================================
|
||||
|
||||
// Abort the kernel when the exact fallback cannot complete, instead of
|
||||
// returning a truncated row. Compile-time so it costs nothing when off; the
|
||||
// counters below record the fault either way, so a build with the trap off is
|
||||
// still never silently wrong.
|
||||
#ifndef MERLIN_TOPK_EXACT_TRAP
|
||||
#define MERLIN_TOPK_EXACT_TRAP 1
|
||||
#endif
|
||||
|
||||
// Device-global counters. Plain device atomics on a process-lifetime buffer:
|
||||
// no host sync and no host-visible allocation on the launch path, so this is
|
||||
// safe to record inside a cudagraph capture and on every replay. Read from
|
||||
// Python with `torch.ops._C.persistent_topk_exact_counters`.
|
||||
enum TopKExactCounter : int {
|
||||
kCtrMediumRows = 0, // denominator: medium-path rows entered
|
||||
kCtrMediumOverflow = 1, // rows where a 4,096-slot guard would drop
|
||||
kCtrMediumExactFallback = 2, // rows that took the exact restart
|
||||
kCtrMediumExactFault = 3, // exact restart could not complete
|
||||
kNumTopKExactCounters = 4,
|
||||
};
|
||||
|
||||
// Exact, bounded-memory re-selection for a single medium-path row.
|
||||
//
|
||||
// `scores` is already offset to the row. Every radix round rescans the whole
|
||||
// row, so correctness does not depend on how many values share a coarse bin.
|
||||
// Entered only after an overflow has been detected, which is 0.1-0.2 % of
|
||||
// medium rows at 38K-51K prompt tokens.
|
||||
template <int TopK>
|
||||
__device__ __noinline__ void medium_exact_restart(
|
||||
const float* __restrict__ scores, int32_t* __restrict__ output,
|
||||
uint32_t length, unsigned int* __restrict__ counters) {
|
||||
static_assert(kThreadsPerBlock >= RADIX,
|
||||
"the single-thread bin walk assumes RADIX <= blockDim.x");
|
||||
|
||||
using BlockScan = cub::BlockScan<uint32_t, kThreadsPerBlock>;
|
||||
|
||||
struct ExactSmem {
|
||||
uint32_t histogram[RADIX];
|
||||
typename BlockScan::TempStorage scan;
|
||||
uint32_t prefix; // ordered-uint radix prefix, grows 8 bits per round
|
||||
uint32_t remaining; // how many of the pivot-equal group must be kept
|
||||
uint32_t tie_total; // how many elements have key == pivot in total
|
||||
uint32_t tie_cut; // lowest element index kept among the pivot-equal
|
||||
uint32_t fault; // set by the walks; block-wide, read after a barrier
|
||||
};
|
||||
|
||||
// The medium path's dynamic smem window is kSmemMedium (35,968 B) and the
|
||||
// buffered state in it is dead the moment we restart, so we overlay it.
|
||||
// Declared with an explicit alignment rather than inheriting the `char`
|
||||
// array's alignment of 1 -- cub::BlockScan::TempStorage needs more.
|
||||
extern __shared__ __align__(16) char medium_exact_smem[];
|
||||
static_assert(alignof(ExactSmem) <= 16, "ExactSmem over-aligned for smem");
|
||||
static_assert(sizeof(ExactSmem) <= kSmemMedium,
|
||||
"exact restart does not fit the medium smem window");
|
||||
|
||||
auto* smem = reinterpret_cast<ExactSmem*>(medium_exact_smem);
|
||||
const uint32_t tx = threadIdx.x;
|
||||
|
||||
if (tx == 0) {
|
||||
smem->prefix = 0;
|
||||
smem->remaining = static_cast<uint32_t>(TopK);
|
||||
smem->tie_total = 0;
|
||||
smem->tie_cut = 0;
|
||||
// The index-space tie walk below is two 8-bit rounds, so it resolves
|
||||
// indices below 65,536. The medium path serves seq_len <= 32,768, but
|
||||
// fail loudly rather than silently mis-cut if that ever changes.
|
||||
smem->fault = (length > 65536u || length < static_cast<uint32_t>(TopK))
|
||||
? 1u
|
||||
: 0u;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// ---- four 8-bit radix rounds over the full row -------------------------
|
||||
#pragma unroll
|
||||
for (uint32_t round = 0; round < 4; ++round) {
|
||||
// An entry fault (bad length) previously still paid four
|
||||
// full-row rescans before anyone looked at the flag. Block-uniform --
|
||||
// the flag is only ever written before a barrier that precedes this read.
|
||||
if (smem->fault != 0u) break;
|
||||
for (uint32_t bin = tx; bin < RADIX; bin += kThreadsPerBlock) {
|
||||
smem->histogram[bin] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Table, not `~0u << (32 - round * 8)`: the round-0 form of that
|
||||
// expression is a 32-bit shift of a 32-bit type, which is undefined even
|
||||
// though the ternary never selects it at runtime, and the loop is fully
|
||||
// unrolled so the compiler does constant-fold the dead branch.
|
||||
constexpr uint32_t kPrefixMask[4] = {0u, 0xFF000000u, 0xFFFF0000u,
|
||||
0xFFFFFF00u};
|
||||
const uint32_t shift = 24u - round * 8u;
|
||||
const uint32_t prefix_mask = kPrefixMask[round];
|
||||
const uint32_t prefix = smem->prefix;
|
||||
|
||||
for (uint32_t idx = tx; idx < length; idx += kThreadsPerBlock) {
|
||||
const uint32_t ordered = convert_to_uint32_v2(scores[idx]);
|
||||
if ((ordered & prefix_mask) == prefix) {
|
||||
atomicAdd(&smem->histogram[(ordered >> shift) & 0xFFu], 1u);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Single-threaded descending walk. #52149's wide-radix variant does this
|
||||
// cooperatively and reads `remaining` in every thread before one thread
|
||||
// writes it back with no barrier in between (a lost update on a
|
||||
// non-atomic shared `|=`); one thread owning both the read and the write
|
||||
// removes that class of bug rather than adding the missing barrier.
|
||||
if (tx == 0) {
|
||||
const uint32_t remaining = smem->remaining;
|
||||
uint32_t count_above = 0;
|
||||
bool found = false;
|
||||
for (int bin = RADIX - 1; bin >= 0; --bin) {
|
||||
const uint32_t count = smem->histogram[bin];
|
||||
if (count_above + count >= remaining) {
|
||||
smem->remaining = remaining - count_above;
|
||||
smem->prefix |= static_cast<uint32_t>(bin) << shift;
|
||||
if (round == 3) smem->tie_total = count;
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
count_above += count;
|
||||
}
|
||||
// No crossing bin means the entry invariant (length >= TopK) is broken.
|
||||
if (!found) atomicOr(&smem->fault, 1u);
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
const uint32_t pivot = smem->prefix;
|
||||
const uint32_t equal_count = smem->remaining;
|
||||
const uint32_t tie_total = smem->tie_total;
|
||||
|
||||
// `> pivot` holds exactly TopK - equal_count elements, so the selection is
|
||||
// TopK elements exactly. equal_count <= tie_total is an invariant of the
|
||||
// walk (count >= remaining - count_above at the crossing bin).
|
||||
if (tx == 0 && (equal_count == 0u || equal_count > tie_total)) {
|
||||
atomicOr(&smem->fault, 1u);
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Note: a fault raised before emission short-circuits ALL
|
||||
// selection work -- no tie walk, no scan, no candidate index ever written
|
||||
// -- so the row goes straight to the -1 fill below. Previously the scan
|
||||
// ran and could write candidates that the fill then overwrote: correct in
|
||||
// the end, but wasted work on the fault path and harder to debug. The
|
||||
// condition is block-uniform (shared, read after the barrier above).
|
||||
if (smem->fault == 0u) {
|
||||
// ---- tie cut at the k-th value: most-recent wins --------------------
|
||||
//
|
||||
// Keep the `equal_count` HIGHEST element indices among the pivot-equal
|
||||
// group. Finding the cut index turns the tie rule into a per-element
|
||||
// predicate, which is what lets the emission below be atomic-free.
|
||||
//
|
||||
// Exact 32-bit ties at the cutoff are rare (19 occurrences across
|
||||
// 12 requests, one element each), so the common case is tie_total ==
|
||||
// equal_count and the whole group is kept with no extra pass.
|
||||
if (tie_total > equal_count) {
|
||||
uint32_t need = equal_count;
|
||||
uint32_t high_byte = 0;
|
||||
|
||||
// Round A: bucket the pivot-equal indices by their high byte.
|
||||
for (uint32_t bin = tx; bin < RADIX; bin += kThreadsPerBlock) {
|
||||
smem->histogram[bin] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (uint32_t idx = tx; idx < length; idx += kThreadsPerBlock) {
|
||||
if (convert_to_uint32_v2(scores[idx]) == pivot) {
|
||||
atomicAdd(&smem->histogram[(idx >> 8) & 0xFFu], 1u);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (tx == 0) {
|
||||
uint32_t count_above = 0;
|
||||
bool found = false;
|
||||
for (int bin = RADIX - 1; bin >= 0; --bin) {
|
||||
const uint32_t count = smem->histogram[bin];
|
||||
if (count_above + count >= need) {
|
||||
smem->tie_cut = static_cast<uint32_t>(bin); // high byte, for now
|
||||
smem->remaining = need - count_above; // carried into round B
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
count_above += count;
|
||||
}
|
||||
if (!found) atomicOr(&smem->fault, 1u);
|
||||
}
|
||||
__syncthreads();
|
||||
high_byte = smem->tie_cut;
|
||||
need = smem->remaining;
|
||||
|
||||
// Round B: within that bucket, bucket by the low byte. Each (high, low)
|
||||
// pair names one index, so every count here is 0 or 1 and the walk lands
|
||||
// exactly on the need-th largest index.
|
||||
for (uint32_t bin = tx; bin < RADIX; bin += kThreadsPerBlock) {
|
||||
smem->histogram[bin] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
for (uint32_t idx = tx; idx < length; idx += kThreadsPerBlock) {
|
||||
if (((idx >> 8) & 0xFFu) == high_byte &&
|
||||
convert_to_uint32_v2(scores[idx]) == pivot) {
|
||||
atomicAdd(&smem->histogram[idx & 0xFFu], 1u);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
if (tx == 0) {
|
||||
uint32_t count_above = 0;
|
||||
bool found = false;
|
||||
for (int bin = RADIX - 1; bin >= 0; --bin) {
|
||||
const uint32_t count = smem->histogram[bin];
|
||||
if (count_above + count >= need) {
|
||||
smem->tie_cut = (high_byte << 8) | static_cast<uint32_t>(bin);
|
||||
found = true;
|
||||
break;
|
||||
}
|
||||
count_above += count;
|
||||
}
|
||||
if (!found) atomicOr(&smem->fault, 1u);
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
}
|
||||
|
||||
// Re-test AFTER the tie walks. The guard above was
|
||||
// entered before them, so a fault raised at their bin-walk failure sites
|
||||
// used to fall straight through into emission. No barrier is needed
|
||||
// between the two tests: the tie walks write the flag from tx == 0 and
|
||||
// each is followed by a barrier inside the block above.
|
||||
if (smem->fault == 0u) {
|
||||
const uint32_t tie_cut = smem->tie_cut;
|
||||
|
||||
// ---- emission: ascending index order, no atomics ---------------------
|
||||
//
|
||||
// selected(i) = ordered(i) > pivot || (ordered(i) == pivot && i >= tie_cut)
|
||||
//
|
||||
// is a pure function of the element, so the output slot is a prefix sum
|
||||
// over the row rather than an arrival rank. Tiles run in ascending base
|
||||
// and within a tile thread `tx` owns `base + tx`, so output[] comes out
|
||||
// strictly ascending in index -- bit-reproducible run to run, and a no-op
|
||||
// for the Python-side MERLIN_QSA_SORT rather than something it corrects.
|
||||
uint32_t base_rank = 0;
|
||||
const uint32_t tiles =
|
||||
(length + kThreadsPerBlock - 1u) / kThreadsPerBlock * kThreadsPerBlock;
|
||||
for (uint32_t base = 0; base < tiles; base += kThreadsPerBlock) {
|
||||
const uint32_t idx = base + tx;
|
||||
uint32_t flag = 0;
|
||||
if (idx < length) {
|
||||
const uint32_t ordered = convert_to_uint32_v2(scores[idx]);
|
||||
flag =
|
||||
(ordered > pivot || (ordered == pivot && idx >= tie_cut)) ? 1u : 0u;
|
||||
}
|
||||
uint32_t rank = 0;
|
||||
uint32_t total = 0;
|
||||
BlockScan(smem->scan).ExclusiveSum(flag, rank, total);
|
||||
if (flag != 0u) {
|
||||
const uint32_t pos = base_rank + rank;
|
||||
// Bounded on both emission phases -- #52149 bounds
|
||||
// `collect_pivot_matches` but not its fused `> pivot` write, so an
|
||||
// invariant break there becomes an out-of-bounds store rather than a
|
||||
// detected fault.
|
||||
if (pos < static_cast<uint32_t>(TopK)) {
|
||||
output[pos] = static_cast<int32_t>(idx);
|
||||
} else {
|
||||
// This is the one fault site with concurrent
|
||||
// writers. Equal stored values do not make a data race a
|
||||
// protocol, so make it an atomic. The tx==0 sites below and
|
||||
// above are single-writer and separated by barriers, but they use
|
||||
// the same primitive so the flag has one access discipline.
|
||||
atomicOr(&smem->fault, 1u);
|
||||
}
|
||||
}
|
||||
base_rank += total;
|
||||
__syncthreads(); // BlockScan TempStorage is reused next tile
|
||||
}
|
||||
|
||||
if (tx == 0 && base_rank != static_cast<uint32_t>(TopK)) {
|
||||
atomicOr(&smem->fault, 1u);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
// Consume the decision into a register and barrier again
|
||||
// BEFORE anyone can leave. The persistent kernel reuses this same dynamic
|
||||
// shared-memory window for the next row -- the large-row loader writes into
|
||||
// it before its first barrier -- so a fast warp returning here while a slow
|
||||
// warp still had to read the shared flag would let the next row's writes
|
||||
// decide this row's fault branch. "Read after a barrier" is uniform only
|
||||
// until the storage is reused; this closes that window. After the second
|
||||
// barrier every thread already holds its copy, so no later write can
|
||||
// change any thread's decision, and the branch below is register-uniform.
|
||||
const uint32_t row_fault = smem->fault;
|
||||
__syncthreads();
|
||||
|
||||
// ---- fail loud ---------------------------------------------------------
|
||||
if (row_fault != 0u) {
|
||||
if (tx == 0 && counters != nullptr) {
|
||||
atomicAdd(&counters[kCtrMediumExactFault], 1u);
|
||||
}
|
||||
// -1 is the kernel's own padding value and the consumer rejects it
|
||||
// (`valid = ... & (token >= 0)`), so a faulted row attends nothing and is
|
||||
// visible downstream instead of quietly attending the wrong blocks.
|
||||
for (uint32_t i = tx; i < static_cast<uint32_t>(TopK);
|
||||
i += kThreadsPerBlock) {
|
||||
output[i] = -1;
|
||||
}
|
||||
__threadfence();
|
||||
__syncthreads();
|
||||
#if MERLIN_TOPK_EXACT_TRAP
|
||||
__trap();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Medium path: coarse FP16 histogram + 4-pass FP32 radix refinement
|
||||
// For sequences 8K < seq_len <= 64K.
|
||||
@@ -424,7 +775,10 @@ __device__ __noinline__ void histogram_2048_topk(
|
||||
template <int TopK>
|
||||
__device__ __noinline__ void histogram_256_topk(
|
||||
const float* __restrict__ logits, int* __restrict__ output_indices,
|
||||
int logits_offset, int seq_len) {
|
||||
int logits_offset, int seq_len, bool exact_medium,
|
||||
unsigned int* __restrict__ counters) {
|
||||
// No default arguments on purpose: a new call site must state whether it
|
||||
// wants the exact path, rather than silently inheriting the broken one.
|
||||
// All shared state lives in dynamic shared memory to avoid static
|
||||
extern __shared__ char medium_smem[];
|
||||
|
||||
@@ -435,6 +789,9 @@ __device__ __noinline__ void histogram_256_topk(
|
||||
int& shared_threshold_bin = medium_scalars[1];
|
||||
int* shared_buffered_count = &medium_scalars[2];
|
||||
int& shared_final_k = medium_scalars[4];
|
||||
// One extra scalar so an overflowing row is counted once, not once per
|
||||
// refinement pass. kMediumScalarsBytes covers it (see the constant).
|
||||
int& shared_overflow_counted = medium_scalars[5];
|
||||
int (*buffered_indices)[MAX_BUFFERED_ITEMS] =
|
||||
reinterpret_cast<int (*)[MAX_BUFFERED_ITEMS]>(medium_smem +
|
||||
kMediumHeaderSize);
|
||||
@@ -442,6 +799,11 @@ __device__ __noinline__ void histogram_256_topk(
|
||||
const int thread_id = threadIdx.x;
|
||||
int remaining_k = TopK;
|
||||
|
||||
if (thread_id == 0) {
|
||||
shared_overflow_counted = 0;
|
||||
if (counters != nullptr) atomicAdd(&counters[kCtrMediumRows], 1u);
|
||||
}
|
||||
|
||||
if (thread_id < RADIX + 1) {
|
||||
shared_histogram[0][thread_id] = 0;
|
||||
}
|
||||
@@ -475,8 +837,25 @@ __device__ __noinline__ void histogram_256_topk(
|
||||
if (thread_id < RADIX && shared_histogram[0][thread_id] > remaining_k &&
|
||||
shared_histogram[0][thread_id + 1] <= remaining_k) {
|
||||
shared_threshold_bin = thread_id;
|
||||
shared_buffered_count[0] = 0;
|
||||
// Medium-path restart / guard site M1 (was `:512`). shared_histogram[0] is in suffix-sum
|
||||
// form here -- it is what the predicate just above tests -- so
|
||||
// hist[t] - hist[t+1] is the EXACT population of the threshold bin, which
|
||||
// is exactly what the collection loop below would buffer. The prediction
|
||||
// is precise, not conservative, so no row pays the exact path unnecessarily
|
||||
// and no overflowing row escapes it.
|
||||
const int threshold_bin_count =
|
||||
shared_histogram[0][thread_id] - shared_histogram[0][thread_id + 1];
|
||||
const bool will_overflow = threshold_bin_count > MAX_BUFFERED_ITEMS;
|
||||
// Sentinel: the refinement loop's entry check reads this and cannot tell
|
||||
// a predicted overflow from a measured one, so both take the same path.
|
||||
shared_buffered_count[0] =
|
||||
(will_overflow && exact_medium) ? MAX_BUFFERED_ITEMS + 1 : 0;
|
||||
shared_output_count = 0;
|
||||
// Flag only -- the counter is incremented past the early return below.
|
||||
// When hist[t+1] == TopK exactly the coarse pass already resolves the row
|
||||
// and the threshold bin is never collected, so a large bin population is
|
||||
// NOT an overflow and must not be counted as one.
|
||||
shared_overflow_counted = will_overflow ? 1 : 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
@@ -495,25 +874,51 @@ __device__ __noinline__ void histogram_256_topk(
|
||||
return;
|
||||
}
|
||||
|
||||
// Past the early return the threshold bin WILL be collected, so the M1
|
||||
// prediction is now a real overflow event.
|
||||
if (thread_id == 0 && shared_overflow_counted == 1 && counters != nullptr) {
|
||||
atomicAdd(&counters[kCtrMediumOverflow], 1u);
|
||||
}
|
||||
|
||||
__syncthreads();
|
||||
if (thread_id < RADIX + 1) {
|
||||
shared_histogram[0][thread_id] = 0;
|
||||
}
|
||||
__syncthreads();
|
||||
|
||||
for (int idx = thread_id; idx < seq_len; idx += kThreadsPerBlock) {
|
||||
const float logit_value = logits[idx + logits_offset];
|
||||
const int bin = convert_to_uint8(logit_value);
|
||||
if (bin > threshold_bin) {
|
||||
const int output_pos = atomicAdd(&shared_output_count, 1);
|
||||
output_indices[output_pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
const int buffer_pos = atomicAdd(&shared_buffered_count[0], 1);
|
||||
if (__builtin_expect(buffer_pos < MAX_BUFFERED_ITEMS, 1)) {
|
||||
buffered_indices[0][buffer_pos] = idx;
|
||||
const uint32_t fp32_bits = convert_to_uint32_v2(logit_value);
|
||||
const int next_bin = (fp32_bits >> 24) & 0xFF;
|
||||
atomicAdd(&shared_histogram[0][next_bin], 1);
|
||||
// On a predicted overflow the collection pass is skipped whole --
|
||||
// including its `bin > threshold_bin` emissions. That matters: the exact
|
||||
// restart rewrites the entire row from slot 0, so anything emitted here
|
||||
// would be double-counted. Skipping is also why the detector cannot ship
|
||||
// without the restart -- on its own it would emit nothing, which is a
|
||||
// fail-loud half, never a silently-truncated one.
|
||||
//
|
||||
// The entry predicate must be IMMUTABLE. It used to read
|
||||
// `shared_buffered_count[0]`, which the loop body then increments -- so with
|
||||
// exact_medium = false (the control arm, where the count starts at 0 and a
|
||||
// real overflow pushes it past capacity mid-loop) a late warp could evaluate
|
||||
// the predicate after early warps had driven the count over 4096 and skip
|
||||
// ALL of its elements, including above-threshold winners. That made `=0` a
|
||||
// new timing-dependent truncation rather than the v2.0.1 baseline.
|
||||
// `shared_overflow_counted` is the M1 prediction, written once before the
|
||||
// barrier above and never touched here; combined with the kernel parameter
|
||||
// it is block-uniform and constant for the whole loop.
|
||||
const bool skip_collection = exact_medium && (shared_overflow_counted == 1);
|
||||
if (__builtin_expect(!skip_collection, 1)) {
|
||||
for (int idx = thread_id; idx < seq_len; idx += kThreadsPerBlock) {
|
||||
const float logit_value = logits[idx + logits_offset];
|
||||
const int bin = convert_to_uint8(logit_value);
|
||||
if (bin > threshold_bin) {
|
||||
const int output_pos = atomicAdd(&shared_output_count, 1);
|
||||
output_indices[output_pos] = idx;
|
||||
} else if (bin == threshold_bin) {
|
||||
const int buffer_pos = atomicAdd(&shared_buffered_count[0], 1);
|
||||
if (__builtin_expect(buffer_pos < MAX_BUFFERED_ITEMS, 1)) {
|
||||
buffered_indices[0][buffer_pos] = idx;
|
||||
const uint32_t fp32_bits = convert_to_uint32_v2(logit_value);
|
||||
const int next_bin = (fp32_bits >> 24) & 0xFF;
|
||||
atomicAdd(&shared_histogram[0][next_bin], 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -524,6 +929,27 @@ __device__ __noinline__ void histogram_256_topk(
|
||||
const int src_buffer = pass % 2;
|
||||
const int dst_buffer = src_buffer ^ 1;
|
||||
const int raw_buffered = shared_buffered_count[src_buffer];
|
||||
// Medium-path restart / guard site M2 (was `:582`) and the M1 sentinel land here. The
|
||||
// append's atomicAdd sits OUTSIDE the capacity guard, so raw_buffered
|
||||
// keeps counting past capacity and a post-hoc test is exact. pass == 3
|
||||
// emits through shared_final_k without buffering, so every buffer filled
|
||||
// at passes 0-2 is read back here: there is no last-pass hole.
|
||||
if (__builtin_expect(raw_buffered > MAX_BUFFERED_ITEMS, 0)) {
|
||||
if (thread_id == 0 && shared_overflow_counted == 0) {
|
||||
shared_overflow_counted = 1;
|
||||
if (counters != nullptr) atomicAdd(&counters[kCtrMediumOverflow], 1u);
|
||||
}
|
||||
if (exact_medium) {
|
||||
if (thread_id == 0 && counters != nullptr) {
|
||||
atomicAdd(&counters[kCtrMediumExactFallback], 1u);
|
||||
}
|
||||
__syncthreads(); // no thread may clobber medium_smem before this
|
||||
medium_exact_restart<TopK>(logits + logits_offset, output_indices,
|
||||
static_cast<uint32_t>(seq_len), counters);
|
||||
return;
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
const int num_buffered =
|
||||
(raw_buffered < MAX_BUFFERED_ITEMS) ? raw_buffered : MAX_BUFFERED_ITEMS;
|
||||
|
||||
@@ -940,7 +1366,8 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
|
||||
} else if (seq_len <= static_cast<uint32_t>(HIST2048_THRESHOLD)) {
|
||||
histogram_2048_topk<TopK>(row_input, row_output, seq_len);
|
||||
} else {
|
||||
histogram_256_topk<TopK>(row_input, row_output, 0, seq_len);
|
||||
histogram_256_topk<TopK>(row_input, row_output, 0, seq_len,
|
||||
params.exact_medium, params.exact_counters);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <algorithm>
|
||||
#include <atomic>
|
||||
#include <cstdlib>
|
||||
#include <mutex>
|
||||
|
||||
#include "ops.h"
|
||||
#include "torch_utils.h"
|
||||
@@ -15,6 +18,140 @@
|
||||
namespace {
|
||||
|
||||
#ifndef USE_ROCM
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// MERLIN_TOPK_EXACT_MEDIUM switch and its counters.
|
||||
//
|
||||
// The switch is resolved ONCE per process, not per launch, so it is a launch
|
||||
// constant: a cudagraph captures the value that was live at capture time and
|
||||
// every replay agrees with it. Default ON in this release; set
|
||||
// MERLIN_TOPK_EXACT_MEDIUM=0 for the A/B control arm, which restores v2.0.1
|
||||
// behaviour exactly (the counters still record, so the control arm reports the
|
||||
// overflow incidence it is the control for).
|
||||
// ---------------------------------------------------------------------------
|
||||
bool exact_medium_enabled() {
|
||||
static const bool enabled = [] {
|
||||
const char* raw = std::getenv("MERLIN_TOPK_EXACT_MEDIUM");
|
||||
return (raw == nullptr || raw[0] == '\0') ? true : (std::atoi(raw) != 0);
|
||||
}();
|
||||
return enabled;
|
||||
}
|
||||
|
||||
// Process-lifetime device counters, one block per device. The kernel only
|
||||
// ever atomicAdds into them, so the launch path needs no host sync and the
|
||||
// values accumulate across cudagraph replays.
|
||||
//
|
||||
// Allocation, however, is NOT capture-safe: cudaMalloc during stream capture
|
||||
// is at best not captured and at worst invalidates the capture. So:
|
||||
// - the allocation only ever happens on a call that is explicitly allowed to
|
||||
// allocate, which is a non-capturing launch or the reader op;
|
||||
// - a call made *during* capture never allocates and never latches a result,
|
||||
// so a capture that happens to be first does not permanently poison the
|
||||
// instrument;
|
||||
// - the launch path fails loudly if it is capturing with no counters, rather
|
||||
// than baking a graph that runs the patched kernel with no instrument at
|
||||
// all. A silently uninstrumented graph is the failure this is here to
|
||||
// prevent, so the check is deliberately not a warning.
|
||||
// Read (and, on the first call, allocate) with
|
||||
// torch.ops._C.persistent_topk_exact_counters, which is the explicit
|
||||
// pre-capture init path.
|
||||
constexpr int kMaxCounterDevices = 16;
|
||||
|
||||
enum CounterState : int {
|
||||
kCountersUninit = 0,
|
||||
kCountersReady = 1,
|
||||
kCountersFailed = 2,
|
||||
};
|
||||
|
||||
// Atomics, not plain globals: the fast path reads these without the mutex, and
|
||||
// a second host thread may be publishing them from the slow path. Release on
|
||||
// publish, acquire on read, and the pointer is always published BEFORE the
|
||||
// state that advertises it.
|
||||
std::atomic<unsigned int*> g_exact_counters[kMaxCounterDevices] = {};
|
||||
std::atomic<int> g_exact_counters_state[kMaxCounterDevices] = {};
|
||||
std::mutex g_exact_counters_mutex;
|
||||
|
||||
// Lookup only. Never allocates, never blocks, safe from any context.
|
||||
unsigned int* get_exact_counters(int device_index) {
|
||||
if (device_index < 0 || device_index >= kMaxCounterDevices) return nullptr;
|
||||
if (g_exact_counters_state[device_index].load(std::memory_order_acquire) !=
|
||||
kCountersReady) {
|
||||
return nullptr;
|
||||
}
|
||||
return g_exact_counters[device_index].load(std::memory_order_acquire);
|
||||
}
|
||||
|
||||
// Allocate, zero, and publish. MUST NOT be called on a capturing stream, and
|
||||
// should be called in a quiescent lifecycle phase: it allocates and blocks,
|
||||
// and neither is safe while any thread in the process is capturing a graph.
|
||||
// `stream_is_capturing` only proves the CALLING stream is idle of capture, so
|
||||
// the intended entry point is the reader op called explicitly before capture;
|
||||
// the non-capturing launch path calls this as a fallback.
|
||||
unsigned int* ensure_exact_counters(int device_index, cudaStream_t stream) {
|
||||
if (device_index < 0 || device_index >= kMaxCounterDevices) return nullptr;
|
||||
|
||||
const int state =
|
||||
g_exact_counters_state[device_index].load(std::memory_order_acquire);
|
||||
if (state == kCountersReady) {
|
||||
return g_exact_counters[device_index].load(std::memory_order_acquire);
|
||||
}
|
||||
// A previous attempt failed for real (OOM); do not retry on every launch.
|
||||
// Only a call that was actually allowed to allocate can reach this state, so
|
||||
// a capture-time call never latches it.
|
||||
if (state == kCountersFailed) return nullptr;
|
||||
|
||||
std::lock_guard<std::mutex> lock(g_exact_counters_mutex);
|
||||
// Re-check under the lock; another thread may have published meanwhile.
|
||||
const int locked_state =
|
||||
g_exact_counters_state[device_index].load(std::memory_order_relaxed);
|
||||
if (locked_state != kCountersUninit) {
|
||||
return (locked_state == kCountersReady)
|
||||
? g_exact_counters[device_index].load(std::memory_order_acquire)
|
||||
: nullptr;
|
||||
}
|
||||
|
||||
constexpr size_t bytes =
|
||||
vllm::persistent::kNumTopKExactCounters * sizeof(unsigned int);
|
||||
unsigned int* ptr = nullptr;
|
||||
if (cudaMalloc(&ptr, bytes) == cudaSuccess && ptr != nullptr) {
|
||||
// Host acquire/release orders the HOST state word; it says
|
||||
// nothing about the device. cudaMemset on device memory is asynchronous
|
||||
// with respect to the host, so publishing Ready straight after it would
|
||||
// let a first launch -- on this or any other stream -- increment counters
|
||||
// that had not been zeroed yet, or read uninitialised bytes. Zero on a
|
||||
// known stream and WAIT for it, so initialisation is complete before the
|
||||
// pointer becomes visible to any consumer.
|
||||
if (cudaMemsetAsync(ptr, 0, bytes, stream) != cudaSuccess ||
|
||||
cudaStreamSynchronize(stream) != cudaSuccess) {
|
||||
cudaFree(ptr);
|
||||
ptr = nullptr;
|
||||
}
|
||||
} else {
|
||||
ptr = nullptr;
|
||||
}
|
||||
// A failed allocation leaves the error state set; clear it so the launch's
|
||||
// own cudaGetLastError() gate does not report this as a kernel failure.
|
||||
if (ptr == nullptr) cudaGetLastError();
|
||||
|
||||
g_exact_counters[device_index].store(ptr, std::memory_order_release);
|
||||
g_exact_counters_state[device_index].store(
|
||||
ptr != nullptr ? kCountersReady : kCountersFailed,
|
||||
std::memory_order_release);
|
||||
return ptr;
|
||||
}
|
||||
|
||||
// True when the stream is mid-capture, in which case nothing on this path may
|
||||
// allocate. A query failure is treated as "capturing": the conservative
|
||||
// direction is to refuse to allocate, not to risk invalidating a capture.
|
||||
bool stream_is_capturing(cudaStream_t stream) {
|
||||
cudaStreamCaptureStatus status = cudaStreamCaptureStatusNone;
|
||||
if (cudaStreamIsCapturing(stream, &status) != cudaSuccess) {
|
||||
cudaGetLastError(); // do not leak this into the launch's error gate
|
||||
return true;
|
||||
}
|
||||
return status != cudaStreamCaptureStatusNone;
|
||||
}
|
||||
|
||||
template <int TopK>
|
||||
void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
const torch::stable::Tensor& lengths,
|
||||
@@ -218,6 +355,30 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
workspace.mutable_data_ptr<uint8_t>());
|
||||
params.ctas_per_group = ctas_per_group;
|
||||
params.max_seq_len = static_cast<uint32_t>(max_seq_len);
|
||||
params.exact_medium = exact_medium_enabled();
|
||||
// Allocate only off the capture path; in the ordinary engine lifecycle the
|
||||
// eager warmup lands here first, long before capture. The explicit,
|
||||
// quiescent-phase entry point is the reader op.
|
||||
const int device_index = static_cast<int>(logits.get_device_index());
|
||||
params.exact_counters = stream_is_capturing(stream)
|
||||
? get_exact_counters(device_index)
|
||||
: ensure_exact_counters(device_index, stream);
|
||||
// Note: this check is UNCONDITIONAL -- eager as well as
|
||||
// capturing, and in both arms. Previously only a capturing launch was
|
||||
// rejected, so an eager launch ran with nullptr and the reader then
|
||||
// reported [0,0,0,0,1]: an unavailable instrument encoded as healthy zero
|
||||
// counts, which is the exact failure the counters exist to prevent.
|
||||
// MERLIN_TOPK_EXACT_MEDIUM=0 is NOT an escape: the control arm's whole job
|
||||
// is to report the overflow incidence it is the control for, so it needs
|
||||
// the counters just as much.
|
||||
STD_TORCH_CHECK(
|
||||
params.exact_counters != nullptr,
|
||||
"persistent_topk: the medium-path exact-restart counters are unavailable on "
|
||||
"device ", device_index,
|
||||
" (not initialised before cudagraph capture, or the one-time "
|
||||
"allocation failed). Call "
|
||||
"torch.ops._C.persistent_topk_exact_counters(out, 0, 1) once per "
|
||||
"device in a quiescent phase before capture.");
|
||||
|
||||
#define LAUNCH_PERSISTENT(TOPK_VAL, VS) \
|
||||
do { \
|
||||
@@ -294,3 +455,114 @@ void persistent_topk(const torch::stable::Tensor& logits,
|
||||
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||
#endif
|
||||
}
|
||||
|
||||
// Read (and, on first call, allocate) the medium-path exact-restart
|
||||
// counters. This is also the explicit pre-capture init path: call it once per
|
||||
// device before any cudagraph capture.
|
||||
//
|
||||
// `out` is an int32 CUDA tensor with at least 5 elements:
|
||||
// [0] medium-path rows entered (denominator)
|
||||
// [1] rows where a 4,096-slot guard would drop a candidate
|
||||
// [2] rows that took the exact restart (0 in the control arm)
|
||||
// [3] exact restarts that could not complete (must stay 0)
|
||||
// [4] 1 if MERLIN_TOPK_EXACT_MEDIUM is on, else 0
|
||||
//
|
||||
// COHERENCY -- stated precisely, because the earlier wording overclaimed.
|
||||
// With `sync == 0` the snapshot is ordered ONLY against work already enqueued
|
||||
// on the current stream; a kernel running on another stream can land
|
||||
// increments it misses. With `sync != 0` the call synchronizes the device
|
||||
// before the copy AND again after the copy/reset have been issued, so on
|
||||
// return every increment submitted before the call is included and any reset
|
||||
// has COMPLETED. That is still conditional on the CALLER HOLDING EXCLUSIVE
|
||||
// SUBMISSION OWNERSHIP: nothing stops another thread enqueueing work after the
|
||||
// leading synchronize, and no claim of unconditional global coherence is made
|
||||
// here. A stronger contract would need an event every later producer waits
|
||||
// on; this op deliberately implements the simpler half and says so.
|
||||
//
|
||||
// `reset != 0` REQUIRES `sync != 0`. Reset between measurements, never during
|
||||
// one.
|
||||
//
|
||||
// Must not be called during cudagraph capture (it allocates, synchronizes and
|
||||
// issues a host-to-device copy). It IS the explicit, quiescent-phase
|
||||
// initialization path: call it once per device before any capture.
|
||||
void persistent_topk_exact_counters(torch::stable::Tensor& out, int64_t reset,
|
||||
int64_t sync) {
|
||||
#ifndef USE_ROCM
|
||||
STD_TORCH_CHECK(out.is_cuda(), "out must be a CUDA tensor");
|
||||
STD_TORCH_CHECK(out.scalar_type() == torch::headeronly::ScalarType::Int,
|
||||
"out must be int32");
|
||||
// Element count alone is not enough. A CUDA scalar expanded
|
||||
// to five elements (stride 0) satisfies numel but owns one element's
|
||||
// storage, and the five consecutive raw int32 writes below would run past
|
||||
// it; an ordinary strided view would receive the values in the wrong slots.
|
||||
// Require a contiguous 1-D buffer before taking the pointer.
|
||||
STD_TORCH_CHECK(out.dim() == 1, "out must be 1-D");
|
||||
STD_TORCH_CHECK(out.is_contiguous(),
|
||||
"out must be contiguous (expanded or strided views would be "
|
||||
"written out of bounds or into the wrong elements)");
|
||||
STD_TORCH_CHECK(out.numel() >= vllm::persistent::kNumTopKExactCounters + 1,
|
||||
"out must hold at least ",
|
||||
vllm::persistent::kNumTopKExactCounters + 1, " elements");
|
||||
STD_TORCH_CHECK(
|
||||
reset == 0 || sync != 0,
|
||||
"persistent_topk_exact_counters: reset requires sync != 0, otherwise the "
|
||||
"zeroing races in-flight kernel atomics and silently drops events");
|
||||
|
||||
const torch::stable::accelerator::DeviceGuard device_guard(
|
||||
out.get_device_index());
|
||||
const cudaStream_t stream = get_current_cuda_stream();
|
||||
STD_TORCH_CHECK(!stream_is_capturing(stream),
|
||||
"persistent_topk_exact_counters must not be called during "
|
||||
"cudagraph capture");
|
||||
constexpr size_t bytes =
|
||||
vllm::persistent::kNumTopKExactCounters * sizeof(unsigned int);
|
||||
|
||||
int32_t* dst = out.mutable_data_ptr<int32_t>();
|
||||
unsigned int* src =
|
||||
ensure_exact_counters(static_cast<int>(out.get_device_index()), stream);
|
||||
|
||||
// An unavailable buffer used to be reported as four zeros
|
||||
// plus "enabled", i.e. an instrument failure indistinguishable from "no
|
||||
// medium work and no faults". Fail instead.
|
||||
STD_TORCH_CHECK(src != nullptr,
|
||||
"persistent_topk_exact_counters: the counter buffer could "
|
||||
"not be allocated on this device; the instrument is "
|
||||
"unavailable and its absence must not read as zero counts");
|
||||
|
||||
if (sync != 0) {
|
||||
// Everything that could still be adding to the block must have retired
|
||||
// before the snapshot (and before a reset) for the numbers to mean
|
||||
// anything across streams.
|
||||
STD_TORCH_CHECK(cudaDeviceSynchronize() == cudaSuccess,
|
||||
"exact counter synchronize failed");
|
||||
}
|
||||
|
||||
STD_TORCH_CHECK(
|
||||
cudaMemcpyAsync(dst, src, bytes, cudaMemcpyDeviceToDevice, stream) ==
|
||||
cudaSuccess,
|
||||
"exact counter read failed");
|
||||
if (reset != 0) {
|
||||
STD_TORCH_CHECK(cudaMemsetAsync(src, 0, bytes, stream) == cudaSuccess,
|
||||
"exact counter reset failed");
|
||||
}
|
||||
|
||||
// Static storage: the source of an async H2D copy must outlive the copy.
|
||||
static const int32_t enabled = exact_medium_enabled() ? 1 : 0;
|
||||
STD_TORCH_CHECK(
|
||||
cudaMemcpyAsync(dst + vllm::persistent::kNumTopKExactCounters, &enabled,
|
||||
sizeof(int32_t), cudaMemcpyHostToDevice,
|
||||
stream) == cudaSuccess,
|
||||
"exact counter flag write failed");
|
||||
|
||||
if (sync != 0) {
|
||||
// The leading synchronize alone left the copy and the reset
|
||||
// merely ENQUEUED, so the caller could return and immediately race the
|
||||
// outstanding reset from another stream. Complete the handoff before
|
||||
// returning.
|
||||
STD_TORCH_CHECK(cudaDeviceSynchronize() == cudaSuccess,
|
||||
"exact counter completion synchronize failed");
|
||||
}
|
||||
#else
|
||||
STD_TORCH_CHECK(false, "persistent_topk is not supported on ROCm");
|
||||
#endif
|
||||
}
|
||||
|
||||
@@ -586,6 +586,10 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C, ops) {
|
||||
"persistent_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
"Tensor workspace, int k, int max_seq_len) -> ()");
|
||||
|
||||
// Read (and optionally reset) the medium-path exact-restart counters.
|
||||
ops.def(
|
||||
"persistent_topk_exact_counters(Tensor! out, int reset, int sync) -> ()");
|
||||
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.def(
|
||||
"cooperative_topk(Tensor logits, Tensor lengths, Tensor! output, "
|
||||
@@ -858,6 +862,8 @@ STABLE_TORCH_LIBRARY_IMPL(_C, CUDA, ops) {
|
||||
ops.impl("top_k_per_row_prefill", TORCH_BOX(&top_k_per_row_prefill));
|
||||
ops.impl("top_k_per_row_decode", TORCH_BOX(&top_k_per_row_decode));
|
||||
ops.impl("persistent_topk", TORCH_BOX(&persistent_topk));
|
||||
ops.impl("persistent_topk_exact_counters",
|
||||
TORCH_BOX(&persistent_topk_exact_counters));
|
||||
#ifdef VLLM_ENABLE_COOPERATIVE_TOPK
|
||||
ops.impl("cooperative_topk", TORCH_BOX(&cooperative_topk));
|
||||
#endif
|
||||
|
||||
Reference in New Issue
Block a user