[Perf] Apply single-pass min_larger finding and binary search in Triton Top-p path. (#42191)

Signed-off-by: js_park <cakeng@naver.com>
Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
Jongseok Park
2026-06-02 17:57:26 -07:00
committed by GitHub
co-authored by Nick Hill
parent b254e0456c
commit ca17b6b17d
+71 -181
View File
@@ -186,7 +186,7 @@ def _topk_topp_kernel(
# max so the search converges to -inf (no masking).
min_logit = tl.minimum(min_logit, max_logit)
# Second passes: Ternary search for pivots
# Second passes: Ternary search for pivot
num_iters = 0
k_pivot = float("inf")
k_pivots_num = tl.zeros((), dtype=tl.uint32)
@@ -279,6 +279,8 @@ def _topk_topp_kernel(
num_iters += 1
if num_iters >= 18 or tl.abs(min_range - max_range) < 1e-9:
k_pivot = (max_range + min_range) / 2.0
min_larger = min_larger_0
num_min_larger = num_min_larger_0
found_pivot = 1
else:
# If top-k outlier gathering failed, search whole logit space
@@ -286,12 +288,12 @@ def _topk_topp_kernel(
min_range = min_logit
found_pivot = 0
while found_pivot == 0:
k_pivot_0 = (max_range - min_range) * 1.0 / 4.0 + min_range
k_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
k_pivots_num_0 = tl.zeros((), dtype=tl.uint32)
min_larger_0 = float("inf")
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
k_pivot_1 = (max_range - min_range) * 2.0 / 4.0 + min_range
k_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
k_pivots_num_1 = tl.zeros((), dtype=tl.uint32)
min_larger_1 = float("inf")
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
@@ -359,6 +361,8 @@ def _topk_topp_kernel(
num_iters += 1
if num_iters >= 18 or tl.abs(min_range - max_range) < 1e-9:
k_pivot = (max_range + min_range) / 2.0
min_larger = min_larger_0
num_min_larger = num_min_larger_0
found_pivot = 1
duplicate_logit = min_larger
@@ -520,17 +524,15 @@ def _topk_topp_kernel(
# Fifth passes: Search for p_pivot
found_pivot = 0
while found_pivot == 0:
p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
p_pivot_0 = (max_range - min_range) * 0.5 + min_range
p_pivots_sum_0 = 0.0
min_larger_0 = 1.0
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
p_pivots_sum_1 = 0.0
min_larger_1 = 1.0
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
# First pass: Calculate p_pivots_sum and min_larger
# Single fused pass: compute p_pivots_sum,
# min_larger, and num_min_larger together.
# See _update_min_larger_stats for the
# tile-level merge logic.
for i in range(0, search_iters):
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
0, BLOCK_SIZE_TRUNC
@@ -540,52 +542,20 @@ def _topk_topp_kernel(
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
)
p_pivots_sum_0 += tl.sum(
probs_blk * (probs_blk > p_pivot_0)
)
masked_larger_0 = tl.where(
probs_blk > p_pivot_0, probs_blk, 1.0
)
min_larger_0 = tl.minimum(
min_larger_0, tl.min(masked_larger_0)
above_0 = probs_blk > p_pivot_0
p_pivots_sum_0 += tl.sum(probs_blk * above_0)
min_larger_0, num_min_larger_0 = (
_update_min_larger_stats(
probs_blk,
above_0,
min_larger_0,
num_min_larger_0,
1.0,
)
)
p_pivots_sum_1 += tl.sum(
probs_blk * (probs_blk > p_pivot_1)
)
masked_larger_1 = tl.where(
probs_blk > p_pivot_1, probs_blk, 1.0
)
min_larger_1 = tl.minimum(
min_larger_1, tl.min(masked_larger_1)
)
# Second pass: Calculate num_min_larger
for i in range(0, search_iters):
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
0, BLOCK_SIZE_TRUNC
)
mask_n_2 = offs_n < search_range
probs_blk = tl.load(
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
)
num_min_larger_0 += tl.sum(
tl.abs(probs_blk - min_larger_0) < 1e-9
)
num_min_larger_1 += tl.sum(
tl.abs(probs_blk - min_larger_1) < 1e-9
)
# Check if any of the pivots satisfy termination condition
if p_pivots_sum_1 >= p and (
p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p
):
p_pivot = p_pivot_1
min_larger_prob = min_larger_1
num_min_larger = num_min_larger_1
p_pivots_sum = p_pivots_sum_1
found_pivot = 1
# Check if the pivot satisfies termination condition
if p_pivots_sum_0 >= p and (
p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p
):
@@ -596,19 +566,17 @@ def _topk_topp_kernel(
found_pivot = 1
# Update range
if p_pivots_sum_1 > p:
min_range = p_pivot_1
elif p_pivots_sum_0 > p:
if p_pivots_sum_0 > p:
min_range = p_pivot_0
if p_pivots_sum_0 < p:
elif p_pivots_sum_0 < p:
max_range = p_pivot_0
elif p_pivots_sum_1 < p:
max_range = p_pivot_1
num_iters += 1
if (max_range - min_range) < 1e-9 or num_iters >= 18:
p_pivot = (max_range + min_range) / 2.0
min_larger_prob = min_larger_0
num_min_larger = num_min_larger_0
p_pivots_sum = p_pivots_sum_0
found_pivot = 1
duplicate_logit = (
@@ -725,17 +693,15 @@ def _topk_topp_kernel(
found_pivot = 0
while found_pivot == 0:
p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
p_pivot_0 = (max_range - min_range) * 0.5 + min_range
p_pivots_sum_0 = 0.0
min_larger_0 = 1.0
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
p_pivots_sum_1 = 0.0
min_larger_1 = 1.0
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
# First pass: Calculate p_pivots_sum and min_larger
# Single fused pass: compute p_pivots_sum,
# min_larger, and num_min_larger together.
# See _update_min_larger_stats for the
# tile-level merge logic.
for i in range(0, search_iters):
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
0, BLOCK_SIZE_TRUNC
@@ -745,53 +711,18 @@ def _topk_topp_kernel(
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
)
p_pivots_sum_0 += tl.sum(
probs_blk * (probs_blk > p_pivot_0)
)
masked_larger_0 = tl.where(
probs_blk > p_pivot_0, probs_blk, 1.0
)
min_larger_0 = tl.minimum(
min_larger_0, tl.min(masked_larger_0)
above_0 = probs_blk > p_pivot_0
p_pivots_sum_0 += tl.sum(probs_blk * above_0)
min_larger_0, num_min_larger_0 = _update_min_larger_stats(
probs_blk,
above_0,
min_larger_0,
num_min_larger_0,
1.0,
)
p_pivots_sum_1 += tl.sum(
probs_blk * (probs_blk > p_pivot_1)
)
masked_larger_1 = tl.where(
probs_blk > p_pivot_1, probs_blk, 1.0
)
min_larger_1 = tl.minimum(
min_larger_1, tl.min(masked_larger_1)
)
# Second pass: Calculate num_min_larger
for i in range(0, search_iters):
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
0, BLOCK_SIZE_TRUNC
)
mask_n_2 = offs_n < search_range
probs_blk = tl.load(
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
)
num_min_larger_0 += tl.sum(
tl.abs(probs_blk - min_larger_0) < 1e-9
)
num_min_larger_1 += tl.sum(
tl.abs(probs_blk - min_larger_1) < 1e-9
)
# Check if any of the pivots satisfy termination condition
if (
p_pivots_sum_1 >= p
and p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p
):
p_pivot = p_pivot_1
min_larger_prob = min_larger_1
num_min_larger = num_min_larger_1
p_pivots_sum = p_pivots_sum_1
found_pivot = 1
# Check if the pivot satisfies termination condition
if (
p_pivots_sum_0 >= p
and p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p
@@ -803,19 +734,17 @@ def _topk_topp_kernel(
found_pivot = 1
# Update range
if p_pivots_sum_1 > p:
min_range = p_pivot_1
elif p_pivots_sum_0 > p:
if p_pivots_sum_0 > p:
min_range = p_pivot_0
if p_pivots_sum_0 < p:
elif p_pivots_sum_0 < p:
max_range = p_pivot_0
elif p_pivots_sum_1 < p:
max_range = p_pivot_1
num_iters += 1
if (max_range - min_range) < 1e-9 or num_iters >= 18:
p_pivot = (max_range + min_range) / 2.0
min_larger_prob = min_larger_0
num_min_larger = num_min_larger_0
p_pivots_sum = p_pivots_sum_0
found_pivot = 1
else:
# Re-populate the buffer with full softmax probabilities
@@ -832,17 +761,15 @@ def _topk_topp_kernel(
found_pivot = 0
while found_pivot == 0:
p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
p_pivot_0 = (max_range - min_range) * 0.5 + min_range
p_pivots_sum_0 = 0.0
min_larger_0 = 1.0
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
p_pivots_sum_1 = 0.0
min_larger_1 = 1.0
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
# First pass: Calculate p_pivots_sum and min_larger
# Single fused pass: compute p_pivots_sum,
# min_larger, and num_min_larger together.
# See _update_min_larger_stats for the
# tile-level merge logic.
for i in range(0, NUM_TILES):
offs_n = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask_n = offs_n < VOCAB_SIZE
@@ -850,51 +777,18 @@ def _topk_topp_kernel(
BUFFER_ROW + offs_n, mask=mask_n, other=0.0
)
p_pivots_sum_0 += tl.sum(
probs_blk * (probs_blk > p_pivot_0)
)
masked_larger_0 = tl.where(
probs_blk > p_pivot_0, probs_blk, 1.0
)
min_larger_0 = tl.minimum(
min_larger_0, tl.min(masked_larger_0)
above_0 = probs_blk > p_pivot_0
p_pivots_sum_0 += tl.sum(probs_blk * above_0)
min_larger_0, num_min_larger_0 = _update_min_larger_stats(
probs_blk,
above_0,
min_larger_0,
num_min_larger_0,
1.0,
)
p_pivots_sum_1 += tl.sum(
probs_blk * (probs_blk > p_pivot_1)
)
masked_larger_1 = tl.where(
probs_blk > p_pivot_1, probs_blk, 1.0
)
min_larger_1 = tl.minimum(
min_larger_1, tl.min(masked_larger_1)
)
# Second pass: Calculate num_min_larger
for i in range(0, NUM_TILES):
offs_n = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
mask_n = offs_n < VOCAB_SIZE
probs_blk = tl.load(
BUFFER_ROW + offs_n, mask=mask_n, other=0.0
)
num_min_larger_0 += tl.sum(
tl.abs(probs_blk - min_larger_0) < 1e-9
)
num_min_larger_1 += tl.sum(
tl.abs(probs_blk - min_larger_1) < 1e-9
)
# Check if any of the pivots satisfy termination condition
if (
p_pivots_sum_1 >= p
and p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p
):
p_pivot = p_pivot_1
min_larger_prob = min_larger_1
num_min_larger = num_min_larger_1
p_pivots_sum = p_pivots_sum_1
found_pivot = 1
# Check if the pivot satisfies termination condition
if (
p_pivots_sum_0 >= p
and p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p
@@ -906,22 +800,20 @@ def _topk_topp_kernel(
found_pivot = 1
# Update range
if p_pivots_sum_1 > p:
min_range = p_pivot_1
elif p_pivots_sum_0 > p:
if p_pivots_sum_0 > p:
min_range = p_pivot_0
if p_pivots_sum_0 < p:
elif p_pivots_sum_0 < p:
max_range = p_pivot_0
elif p_pivots_sum_1 < p:
max_range = p_pivot_1
num_iters += 1
if (max_range - min_range) < 1e-9 or num_iters >= 18:
p_pivot = (max_range + min_range) / 2.0
min_larger_prob = min_larger_0
num_min_larger = num_min_larger_0
p_pivots_sum = p_pivots_sum_0
found_pivot = 1
duplicate_logit = tl.log(min_larger_prob * sum_exp_logits) + max_logit
duplicate_logit = tl.log(min_larger_prob * sum_exp_logits) + max_sample
num_duplicate_logit = num_min_larger
num_keep = num_duplicate_logit - tl.cast(
(p_pivots_sum - p) / min_larger_prob, tl.uint32
@@ -952,9 +844,7 @@ def _topk_topp_kernel(
tl.abs(logits_blk - duplicate_logit) < 1e-9
) & mask_n
duplicate_count = tl.cumsum(duplicate_mask) + num_kept
duplicate_keep_mask = (
duplicate_count <= num_duplicate_logit
) & duplicate_mask
duplicate_keep_mask = (duplicate_count <= num_keep) & duplicate_mask
duplicate_remove_mask = duplicate_mask & ~duplicate_keep_mask
num_kept += tl.sum(duplicate_keep_mask)
keep_mask = keep_mask & (~duplicate_remove_mask)