[Perf] Apply single-pass min_larger finding and binary search in Triton Top-p path. (#42191)
Signed-off-by: js_park <cakeng@naver.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
This commit is contained in:
co-authored by
Nick Hill
parent
b254e0456c
commit
ca17b6b17d
@@ -186,7 +186,7 @@ def _topk_topp_kernel(
|
||||
# max so the search converges to -inf (no masking).
|
||||
min_logit = tl.minimum(min_logit, max_logit)
|
||||
|
||||
# Second passes: Ternary search for pivots
|
||||
# Second passes: Ternary search for pivot
|
||||
num_iters = 0
|
||||
k_pivot = float("inf")
|
||||
k_pivots_num = tl.zeros((), dtype=tl.uint32)
|
||||
@@ -279,6 +279,8 @@ def _topk_topp_kernel(
|
||||
num_iters += 1
|
||||
if num_iters >= 18 or tl.abs(min_range - max_range) < 1e-9:
|
||||
k_pivot = (max_range + min_range) / 2.0
|
||||
min_larger = min_larger_0
|
||||
num_min_larger = num_min_larger_0
|
||||
found_pivot = 1
|
||||
else:
|
||||
# If top-k outlier gathering failed, search whole logit space
|
||||
@@ -286,12 +288,12 @@ def _topk_topp_kernel(
|
||||
min_range = min_logit
|
||||
found_pivot = 0
|
||||
while found_pivot == 0:
|
||||
k_pivot_0 = (max_range - min_range) * 1.0 / 4.0 + min_range
|
||||
k_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
|
||||
k_pivots_num_0 = tl.zeros((), dtype=tl.uint32)
|
||||
min_larger_0 = float("inf")
|
||||
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
k_pivot_1 = (max_range - min_range) * 2.0 / 4.0 + min_range
|
||||
k_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
|
||||
k_pivots_num_1 = tl.zeros((), dtype=tl.uint32)
|
||||
min_larger_1 = float("inf")
|
||||
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
|
||||
@@ -359,6 +361,8 @@ def _topk_topp_kernel(
|
||||
num_iters += 1
|
||||
if num_iters >= 18 or tl.abs(min_range - max_range) < 1e-9:
|
||||
k_pivot = (max_range + min_range) / 2.0
|
||||
min_larger = min_larger_0
|
||||
num_min_larger = num_min_larger_0
|
||||
found_pivot = 1
|
||||
|
||||
duplicate_logit = min_larger
|
||||
@@ -520,17 +524,15 @@ def _topk_topp_kernel(
|
||||
# Fifth passes: Search for p_pivot
|
||||
found_pivot = 0
|
||||
while found_pivot == 0:
|
||||
p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
|
||||
p_pivot_0 = (max_range - min_range) * 0.5 + min_range
|
||||
p_pivots_sum_0 = 0.0
|
||||
min_larger_0 = 1.0
|
||||
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
|
||||
p_pivots_sum_1 = 0.0
|
||||
min_larger_1 = 1.0
|
||||
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
# First pass: Calculate p_pivots_sum and min_larger
|
||||
# Single fused pass: compute p_pivots_sum,
|
||||
# min_larger, and num_min_larger together.
|
||||
# See _update_min_larger_stats for the
|
||||
# tile-level merge logic.
|
||||
for i in range(0, search_iters):
|
||||
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
|
||||
0, BLOCK_SIZE_TRUNC
|
||||
@@ -540,52 +542,20 @@ def _topk_topp_kernel(
|
||||
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
|
||||
)
|
||||
|
||||
p_pivots_sum_0 += tl.sum(
|
||||
probs_blk * (probs_blk > p_pivot_0)
|
||||
)
|
||||
masked_larger_0 = tl.where(
|
||||
probs_blk > p_pivot_0, probs_blk, 1.0
|
||||
)
|
||||
min_larger_0 = tl.minimum(
|
||||
min_larger_0, tl.min(masked_larger_0)
|
||||
above_0 = probs_blk > p_pivot_0
|
||||
p_pivots_sum_0 += tl.sum(probs_blk * above_0)
|
||||
|
||||
min_larger_0, num_min_larger_0 = (
|
||||
_update_min_larger_stats(
|
||||
probs_blk,
|
||||
above_0,
|
||||
min_larger_0,
|
||||
num_min_larger_0,
|
||||
1.0,
|
||||
)
|
||||
)
|
||||
|
||||
p_pivots_sum_1 += tl.sum(
|
||||
probs_blk * (probs_blk > p_pivot_1)
|
||||
)
|
||||
masked_larger_1 = tl.where(
|
||||
probs_blk > p_pivot_1, probs_blk, 1.0
|
||||
)
|
||||
min_larger_1 = tl.minimum(
|
||||
min_larger_1, tl.min(masked_larger_1)
|
||||
)
|
||||
|
||||
# Second pass: Calculate num_min_larger
|
||||
for i in range(0, search_iters):
|
||||
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
|
||||
0, BLOCK_SIZE_TRUNC
|
||||
)
|
||||
mask_n_2 = offs_n < search_range
|
||||
probs_blk = tl.load(
|
||||
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
|
||||
)
|
||||
|
||||
num_min_larger_0 += tl.sum(
|
||||
tl.abs(probs_blk - min_larger_0) < 1e-9
|
||||
)
|
||||
num_min_larger_1 += tl.sum(
|
||||
tl.abs(probs_blk - min_larger_1) < 1e-9
|
||||
)
|
||||
|
||||
# Check if any of the pivots satisfy termination condition
|
||||
if p_pivots_sum_1 >= p and (
|
||||
p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p
|
||||
):
|
||||
p_pivot = p_pivot_1
|
||||
min_larger_prob = min_larger_1
|
||||
num_min_larger = num_min_larger_1
|
||||
p_pivots_sum = p_pivots_sum_1
|
||||
found_pivot = 1
|
||||
# Check if the pivot satisfies termination condition
|
||||
if p_pivots_sum_0 >= p and (
|
||||
p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p
|
||||
):
|
||||
@@ -596,19 +566,17 @@ def _topk_topp_kernel(
|
||||
found_pivot = 1
|
||||
|
||||
# Update range
|
||||
if p_pivots_sum_1 > p:
|
||||
min_range = p_pivot_1
|
||||
elif p_pivots_sum_0 > p:
|
||||
if p_pivots_sum_0 > p:
|
||||
min_range = p_pivot_0
|
||||
|
||||
if p_pivots_sum_0 < p:
|
||||
elif p_pivots_sum_0 < p:
|
||||
max_range = p_pivot_0
|
||||
elif p_pivots_sum_1 < p:
|
||||
max_range = p_pivot_1
|
||||
|
||||
num_iters += 1
|
||||
if (max_range - min_range) < 1e-9 or num_iters >= 18:
|
||||
p_pivot = (max_range + min_range) / 2.0
|
||||
min_larger_prob = min_larger_0
|
||||
num_min_larger = num_min_larger_0
|
||||
p_pivots_sum = p_pivots_sum_0
|
||||
found_pivot = 1
|
||||
|
||||
duplicate_logit = (
|
||||
@@ -725,17 +693,15 @@ def _topk_topp_kernel(
|
||||
|
||||
found_pivot = 0
|
||||
while found_pivot == 0:
|
||||
p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
|
||||
p_pivot_0 = (max_range - min_range) * 0.5 + min_range
|
||||
p_pivots_sum_0 = 0.0
|
||||
min_larger_0 = 1.0
|
||||
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
|
||||
p_pivots_sum_1 = 0.0
|
||||
min_larger_1 = 1.0
|
||||
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
# First pass: Calculate p_pivots_sum and min_larger
|
||||
# Single fused pass: compute p_pivots_sum,
|
||||
# min_larger, and num_min_larger together.
|
||||
# See _update_min_larger_stats for the
|
||||
# tile-level merge logic.
|
||||
for i in range(0, search_iters):
|
||||
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
|
||||
0, BLOCK_SIZE_TRUNC
|
||||
@@ -745,53 +711,18 @@ def _topk_topp_kernel(
|
||||
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
|
||||
)
|
||||
|
||||
p_pivots_sum_0 += tl.sum(
|
||||
probs_blk * (probs_blk > p_pivot_0)
|
||||
)
|
||||
masked_larger_0 = tl.where(
|
||||
probs_blk > p_pivot_0, probs_blk, 1.0
|
||||
)
|
||||
min_larger_0 = tl.minimum(
|
||||
min_larger_0, tl.min(masked_larger_0)
|
||||
above_0 = probs_blk > p_pivot_0
|
||||
p_pivots_sum_0 += tl.sum(probs_blk * above_0)
|
||||
|
||||
min_larger_0, num_min_larger_0 = _update_min_larger_stats(
|
||||
probs_blk,
|
||||
above_0,
|
||||
min_larger_0,
|
||||
num_min_larger_0,
|
||||
1.0,
|
||||
)
|
||||
|
||||
p_pivots_sum_1 += tl.sum(
|
||||
probs_blk * (probs_blk > p_pivot_1)
|
||||
)
|
||||
masked_larger_1 = tl.where(
|
||||
probs_blk > p_pivot_1, probs_blk, 1.0
|
||||
)
|
||||
min_larger_1 = tl.minimum(
|
||||
min_larger_1, tl.min(masked_larger_1)
|
||||
)
|
||||
|
||||
# Second pass: Calculate num_min_larger
|
||||
for i in range(0, search_iters):
|
||||
offs_n = i * BLOCK_SIZE_TRUNC + tl.arange(
|
||||
0, BLOCK_SIZE_TRUNC
|
||||
)
|
||||
mask_n_2 = offs_n < search_range
|
||||
probs_blk = tl.load(
|
||||
BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0
|
||||
)
|
||||
|
||||
num_min_larger_0 += tl.sum(
|
||||
tl.abs(probs_blk - min_larger_0) < 1e-9
|
||||
)
|
||||
num_min_larger_1 += tl.sum(
|
||||
tl.abs(probs_blk - min_larger_1) < 1e-9
|
||||
)
|
||||
|
||||
# Check if any of the pivots satisfy termination condition
|
||||
if (
|
||||
p_pivots_sum_1 >= p
|
||||
and p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p
|
||||
):
|
||||
p_pivot = p_pivot_1
|
||||
min_larger_prob = min_larger_1
|
||||
num_min_larger = num_min_larger_1
|
||||
p_pivots_sum = p_pivots_sum_1
|
||||
found_pivot = 1
|
||||
# Check if the pivot satisfies termination condition
|
||||
if (
|
||||
p_pivots_sum_0 >= p
|
||||
and p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p
|
||||
@@ -803,19 +734,17 @@ def _topk_topp_kernel(
|
||||
found_pivot = 1
|
||||
|
||||
# Update range
|
||||
if p_pivots_sum_1 > p:
|
||||
min_range = p_pivot_1
|
||||
elif p_pivots_sum_0 > p:
|
||||
if p_pivots_sum_0 > p:
|
||||
min_range = p_pivot_0
|
||||
|
||||
if p_pivots_sum_0 < p:
|
||||
elif p_pivots_sum_0 < p:
|
||||
max_range = p_pivot_0
|
||||
elif p_pivots_sum_1 < p:
|
||||
max_range = p_pivot_1
|
||||
|
||||
num_iters += 1
|
||||
if (max_range - min_range) < 1e-9 or num_iters >= 18:
|
||||
p_pivot = (max_range + min_range) / 2.0
|
||||
min_larger_prob = min_larger_0
|
||||
num_min_larger = num_min_larger_0
|
||||
p_pivots_sum = p_pivots_sum_0
|
||||
found_pivot = 1
|
||||
else:
|
||||
# Re-populate the buffer with full softmax probabilities
|
||||
@@ -832,17 +761,15 @@ def _topk_topp_kernel(
|
||||
|
||||
found_pivot = 0
|
||||
while found_pivot == 0:
|
||||
p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range
|
||||
p_pivot_0 = (max_range - min_range) * 0.5 + min_range
|
||||
p_pivots_sum_0 = 0.0
|
||||
min_larger_0 = 1.0
|
||||
num_min_larger_0 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range
|
||||
p_pivots_sum_1 = 0.0
|
||||
min_larger_1 = 1.0
|
||||
num_min_larger_1 = tl.zeros((), dtype=tl.uint32)
|
||||
|
||||
# First pass: Calculate p_pivots_sum and min_larger
|
||||
# Single fused pass: compute p_pivots_sum,
|
||||
# min_larger, and num_min_larger together.
|
||||
# See _update_min_larger_stats for the
|
||||
# tile-level merge logic.
|
||||
for i in range(0, NUM_TILES):
|
||||
offs_n = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask_n = offs_n < VOCAB_SIZE
|
||||
@@ -850,51 +777,18 @@ def _topk_topp_kernel(
|
||||
BUFFER_ROW + offs_n, mask=mask_n, other=0.0
|
||||
)
|
||||
|
||||
p_pivots_sum_0 += tl.sum(
|
||||
probs_blk * (probs_blk > p_pivot_0)
|
||||
)
|
||||
masked_larger_0 = tl.where(
|
||||
probs_blk > p_pivot_0, probs_blk, 1.0
|
||||
)
|
||||
min_larger_0 = tl.minimum(
|
||||
min_larger_0, tl.min(masked_larger_0)
|
||||
above_0 = probs_blk > p_pivot_0
|
||||
p_pivots_sum_0 += tl.sum(probs_blk * above_0)
|
||||
|
||||
min_larger_0, num_min_larger_0 = _update_min_larger_stats(
|
||||
probs_blk,
|
||||
above_0,
|
||||
min_larger_0,
|
||||
num_min_larger_0,
|
||||
1.0,
|
||||
)
|
||||
|
||||
p_pivots_sum_1 += tl.sum(
|
||||
probs_blk * (probs_blk > p_pivot_1)
|
||||
)
|
||||
masked_larger_1 = tl.where(
|
||||
probs_blk > p_pivot_1, probs_blk, 1.0
|
||||
)
|
||||
min_larger_1 = tl.minimum(
|
||||
min_larger_1, tl.min(masked_larger_1)
|
||||
)
|
||||
|
||||
# Second pass: Calculate num_min_larger
|
||||
for i in range(0, NUM_TILES):
|
||||
offs_n = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
|
||||
mask_n = offs_n < VOCAB_SIZE
|
||||
probs_blk = tl.load(
|
||||
BUFFER_ROW + offs_n, mask=mask_n, other=0.0
|
||||
)
|
||||
|
||||
num_min_larger_0 += tl.sum(
|
||||
tl.abs(probs_blk - min_larger_0) < 1e-9
|
||||
)
|
||||
num_min_larger_1 += tl.sum(
|
||||
tl.abs(probs_blk - min_larger_1) < 1e-9
|
||||
)
|
||||
|
||||
# Check if any of the pivots satisfy termination condition
|
||||
if (
|
||||
p_pivots_sum_1 >= p
|
||||
and p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p
|
||||
):
|
||||
p_pivot = p_pivot_1
|
||||
min_larger_prob = min_larger_1
|
||||
num_min_larger = num_min_larger_1
|
||||
p_pivots_sum = p_pivots_sum_1
|
||||
found_pivot = 1
|
||||
# Check if the pivot satisfies termination condition
|
||||
if (
|
||||
p_pivots_sum_0 >= p
|
||||
and p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p
|
||||
@@ -906,22 +800,20 @@ def _topk_topp_kernel(
|
||||
found_pivot = 1
|
||||
|
||||
# Update range
|
||||
if p_pivots_sum_1 > p:
|
||||
min_range = p_pivot_1
|
||||
elif p_pivots_sum_0 > p:
|
||||
if p_pivots_sum_0 > p:
|
||||
min_range = p_pivot_0
|
||||
|
||||
if p_pivots_sum_0 < p:
|
||||
elif p_pivots_sum_0 < p:
|
||||
max_range = p_pivot_0
|
||||
elif p_pivots_sum_1 < p:
|
||||
max_range = p_pivot_1
|
||||
|
||||
num_iters += 1
|
||||
if (max_range - min_range) < 1e-9 or num_iters >= 18:
|
||||
p_pivot = (max_range + min_range) / 2.0
|
||||
min_larger_prob = min_larger_0
|
||||
num_min_larger = num_min_larger_0
|
||||
p_pivots_sum = p_pivots_sum_0
|
||||
found_pivot = 1
|
||||
|
||||
duplicate_logit = tl.log(min_larger_prob * sum_exp_logits) + max_logit
|
||||
duplicate_logit = tl.log(min_larger_prob * sum_exp_logits) + max_sample
|
||||
num_duplicate_logit = num_min_larger
|
||||
num_keep = num_duplicate_logit - tl.cast(
|
||||
(p_pivots_sum - p) / min_larger_prob, tl.uint32
|
||||
@@ -952,9 +844,7 @@ def _topk_topp_kernel(
|
||||
tl.abs(logits_blk - duplicate_logit) < 1e-9
|
||||
) & mask_n
|
||||
duplicate_count = tl.cumsum(duplicate_mask) + num_kept
|
||||
duplicate_keep_mask = (
|
||||
duplicate_count <= num_duplicate_logit
|
||||
) & duplicate_mask
|
||||
duplicate_keep_mask = (duplicate_count <= num_keep) & duplicate_mask
|
||||
duplicate_remove_mask = duplicate_mask & ~duplicate_keep_mask
|
||||
num_kept += tl.sum(duplicate_keep_mask)
|
||||
keep_mask = keep_mask & (~duplicate_remove_mask)
|
||||
|
||||
Reference in New Issue
Block a user