[Bugfix] Fix persistent_topk inter-CTA init race on RadixRowState (#41444)
Signed-off-by: Yongye Zhu <zyy1102000@gmail.com>
This commit is contained in:
@@ -887,27 +887,14 @@ __global__ void __launch_bounds__(kThreadsPerBlock, 2)
|
||||
uint32_t* shared_ordered =
|
||||
reinterpret_cast<uint32_t*>(smem_raw + kFixedSmemLarge);
|
||||
|
||||
// RadixRowState for multi-CTA cooperative radix
|
||||
// RadixRowState for multi-CTA cooperative radix.
|
||||
// Zero-initialization is done host-side via cudaMemsetAsync in topk.cu
|
||||
// before launch — that gives a stream-ordered happens-before edge for all
|
||||
// CTAs, which the previous in-kernel init (CTA-0 only + intra-CTA
|
||||
// __syncthreads) did not provide and which manifested as a race against
|
||||
// CTA-1+'s first red_release on arrival_counter.
|
||||
RadixRowState* state = ¶ms.row_states[group_id];
|
||||
|
||||
// -- Initialize RadixRowState (only needed if large rows exist) --
|
||||
if (params.max_seq_len > RADIX_THRESHOLD) {
|
||||
if (cta_in_group == 0) {
|
||||
for (uint32_t buf = 0; buf < 3; buf++) {
|
||||
for (uint32_t i = tx; i < RADIX; i += kThreadsPerBlock) {
|
||||
state->histogram[buf][i] = 0;
|
||||
}
|
||||
}
|
||||
if (tx == 0) {
|
||||
state->remaining_k = 0;
|
||||
state->prefix = 0;
|
||||
state->arrival_counter = 0;
|
||||
state->output_counter = 0;
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
int barrier_phase = 0;
|
||||
const uint32_t total_iters = (params.num_rows + num_groups - 1) / num_groups;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user