From ca17b6b17d988bc848d5ca629e677513d3ce6c4a Mon Sep 17 00:00:00 2001 From: Jongseok Park <37990712+cakeng@users.noreply.github.com> Date: Tue, 2 Jun 2026 17:57:26 -0700 Subject: [PATCH] [Perf] Apply single-pass min_larger finding and binary search in Triton Top-p path. (#42191) Signed-off-by: js_park Co-authored-by: Nick Hill --- vllm/v1/sample/ops/topk_topp_triton.py | 252 +++++++------------------ 1 file changed, 71 insertions(+), 181 deletions(-) diff --git a/vllm/v1/sample/ops/topk_topp_triton.py b/vllm/v1/sample/ops/topk_topp_triton.py index 4b17831dfc..bfe6fd6ae5 100644 --- a/vllm/v1/sample/ops/topk_topp_triton.py +++ b/vllm/v1/sample/ops/topk_topp_triton.py @@ -186,7 +186,7 @@ def _topk_topp_kernel( # max so the search converges to -inf (no masking). min_logit = tl.minimum(min_logit, max_logit) - # Second passes: Ternary search for pivots + # Second passes: Ternary search for pivot num_iters = 0 k_pivot = float("inf") k_pivots_num = tl.zeros((), dtype=tl.uint32) @@ -279,6 +279,8 @@ def _topk_topp_kernel( num_iters += 1 if num_iters >= 18 or tl.abs(min_range - max_range) < 1e-9: k_pivot = (max_range + min_range) / 2.0 + min_larger = min_larger_0 + num_min_larger = num_min_larger_0 found_pivot = 1 else: # If top-k outlier gathering failed, search whole logit space @@ -286,12 +288,12 @@ def _topk_topp_kernel( min_range = min_logit found_pivot = 0 while found_pivot == 0: - k_pivot_0 = (max_range - min_range) * 1.0 / 4.0 + min_range + k_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range k_pivots_num_0 = tl.zeros((), dtype=tl.uint32) min_larger_0 = float("inf") num_min_larger_0 = tl.zeros((), dtype=tl.uint32) - k_pivot_1 = (max_range - min_range) * 2.0 / 4.0 + min_range + k_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range k_pivots_num_1 = tl.zeros((), dtype=tl.uint32) min_larger_1 = float("inf") num_min_larger_1 = tl.zeros((), dtype=tl.uint32) @@ -359,6 +361,8 @@ def _topk_topp_kernel( num_iters += 1 if num_iters >= 18 or tl.abs(min_range - max_range) < 1e-9: k_pivot = (max_range + min_range) / 2.0 + min_larger = min_larger_0 + num_min_larger = num_min_larger_0 found_pivot = 1 duplicate_logit = min_larger @@ -520,17 +524,15 @@ def _topk_topp_kernel( # Fifth passes: Search for p_pivot found_pivot = 0 while found_pivot == 0: - p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range + p_pivot_0 = (max_range - min_range) * 0.5 + min_range p_pivots_sum_0 = 0.0 min_larger_0 = 1.0 num_min_larger_0 = tl.zeros((), dtype=tl.uint32) - p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range - p_pivots_sum_1 = 0.0 - min_larger_1 = 1.0 - num_min_larger_1 = tl.zeros((), dtype=tl.uint32) - - # First pass: Calculate p_pivots_sum and min_larger + # Single fused pass: compute p_pivots_sum, + # min_larger, and num_min_larger together. + # See _update_min_larger_stats for the + # tile-level merge logic. for i in range(0, search_iters): offs_n = i * BLOCK_SIZE_TRUNC + tl.arange( 0, BLOCK_SIZE_TRUNC @@ -540,52 +542,20 @@ def _topk_topp_kernel( BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0 ) - p_pivots_sum_0 += tl.sum( - probs_blk * (probs_blk > p_pivot_0) - ) - masked_larger_0 = tl.where( - probs_blk > p_pivot_0, probs_blk, 1.0 - ) - min_larger_0 = tl.minimum( - min_larger_0, tl.min(masked_larger_0) + above_0 = probs_blk > p_pivot_0 + p_pivots_sum_0 += tl.sum(probs_blk * above_0) + + min_larger_0, num_min_larger_0 = ( + _update_min_larger_stats( + probs_blk, + above_0, + min_larger_0, + num_min_larger_0, + 1.0, + ) ) - p_pivots_sum_1 += tl.sum( - probs_blk * (probs_blk > p_pivot_1) - ) - masked_larger_1 = tl.where( - probs_blk > p_pivot_1, probs_blk, 1.0 - ) - min_larger_1 = tl.minimum( - min_larger_1, tl.min(masked_larger_1) - ) - - # Second pass: Calculate num_min_larger - for i in range(0, search_iters): - offs_n = i * BLOCK_SIZE_TRUNC + tl.arange( - 0, BLOCK_SIZE_TRUNC - ) - mask_n_2 = offs_n < search_range - probs_blk = tl.load( - BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0 - ) - - num_min_larger_0 += tl.sum( - tl.abs(probs_blk - min_larger_0) < 1e-9 - ) - num_min_larger_1 += tl.sum( - tl.abs(probs_blk - min_larger_1) < 1e-9 - ) - - # Check if any of the pivots satisfy termination condition - if p_pivots_sum_1 >= p and ( - p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p - ): - p_pivot = p_pivot_1 - min_larger_prob = min_larger_1 - num_min_larger = num_min_larger_1 - p_pivots_sum = p_pivots_sum_1 - found_pivot = 1 + # Check if the pivot satisfies termination condition if p_pivots_sum_0 >= p and ( p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p ): @@ -596,19 +566,17 @@ def _topk_topp_kernel( found_pivot = 1 # Update range - if p_pivots_sum_1 > p: - min_range = p_pivot_1 - elif p_pivots_sum_0 > p: + if p_pivots_sum_0 > p: min_range = p_pivot_0 - - if p_pivots_sum_0 < p: + elif p_pivots_sum_0 < p: max_range = p_pivot_0 - elif p_pivots_sum_1 < p: - max_range = p_pivot_1 num_iters += 1 if (max_range - min_range) < 1e-9 or num_iters >= 18: p_pivot = (max_range + min_range) / 2.0 + min_larger_prob = min_larger_0 + num_min_larger = num_min_larger_0 + p_pivots_sum = p_pivots_sum_0 found_pivot = 1 duplicate_logit = ( @@ -725,17 +693,15 @@ def _topk_topp_kernel( found_pivot = 0 while found_pivot == 0: - p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range + p_pivot_0 = (max_range - min_range) * 0.5 + min_range p_pivots_sum_0 = 0.0 min_larger_0 = 1.0 num_min_larger_0 = tl.zeros((), dtype=tl.uint32) - p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range - p_pivots_sum_1 = 0.0 - min_larger_1 = 1.0 - num_min_larger_1 = tl.zeros((), dtype=tl.uint32) - - # First pass: Calculate p_pivots_sum and min_larger + # Single fused pass: compute p_pivots_sum, + # min_larger, and num_min_larger together. + # See _update_min_larger_stats for the + # tile-level merge logic. for i in range(0, search_iters): offs_n = i * BLOCK_SIZE_TRUNC + tl.arange( 0, BLOCK_SIZE_TRUNC @@ -745,53 +711,18 @@ def _topk_topp_kernel( BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0 ) - p_pivots_sum_0 += tl.sum( - probs_blk * (probs_blk > p_pivot_0) - ) - masked_larger_0 = tl.where( - probs_blk > p_pivot_0, probs_blk, 1.0 - ) - min_larger_0 = tl.minimum( - min_larger_0, tl.min(masked_larger_0) + above_0 = probs_blk > p_pivot_0 + p_pivots_sum_0 += tl.sum(probs_blk * above_0) + + min_larger_0, num_min_larger_0 = _update_min_larger_stats( + probs_blk, + above_0, + min_larger_0, + num_min_larger_0, + 1.0, ) - p_pivots_sum_1 += tl.sum( - probs_blk * (probs_blk > p_pivot_1) - ) - masked_larger_1 = tl.where( - probs_blk > p_pivot_1, probs_blk, 1.0 - ) - min_larger_1 = tl.minimum( - min_larger_1, tl.min(masked_larger_1) - ) - - # Second pass: Calculate num_min_larger - for i in range(0, search_iters): - offs_n = i * BLOCK_SIZE_TRUNC + tl.arange( - 0, BLOCK_SIZE_TRUNC - ) - mask_n_2 = offs_n < search_range - probs_blk = tl.load( - BUFFER_ROW + offs_n, mask=mask_n_2, other=0.0 - ) - - num_min_larger_0 += tl.sum( - tl.abs(probs_blk - min_larger_0) < 1e-9 - ) - num_min_larger_1 += tl.sum( - tl.abs(probs_blk - min_larger_1) < 1e-9 - ) - - # Check if any of the pivots satisfy termination condition - if ( - p_pivots_sum_1 >= p - and p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p - ): - p_pivot = p_pivot_1 - min_larger_prob = min_larger_1 - num_min_larger = num_min_larger_1 - p_pivots_sum = p_pivots_sum_1 - found_pivot = 1 + # Check if the pivot satisfies termination condition if ( p_pivots_sum_0 >= p and p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p @@ -803,19 +734,17 @@ def _topk_topp_kernel( found_pivot = 1 # Update range - if p_pivots_sum_1 > p: - min_range = p_pivot_1 - elif p_pivots_sum_0 > p: + if p_pivots_sum_0 > p: min_range = p_pivot_0 - - if p_pivots_sum_0 < p: + elif p_pivots_sum_0 < p: max_range = p_pivot_0 - elif p_pivots_sum_1 < p: - max_range = p_pivot_1 num_iters += 1 if (max_range - min_range) < 1e-9 or num_iters >= 18: p_pivot = (max_range + min_range) / 2.0 + min_larger_prob = min_larger_0 + num_min_larger = num_min_larger_0 + p_pivots_sum = p_pivots_sum_0 found_pivot = 1 else: # Re-populate the buffer with full softmax probabilities @@ -832,17 +761,15 @@ def _topk_topp_kernel( found_pivot = 0 while found_pivot == 0: - p_pivot_0 = (max_range - min_range) * 1.0 / 3.0 + min_range + p_pivot_0 = (max_range - min_range) * 0.5 + min_range p_pivots_sum_0 = 0.0 min_larger_0 = 1.0 num_min_larger_0 = tl.zeros((), dtype=tl.uint32) - p_pivot_1 = (max_range - min_range) * 2.0 / 3.0 + min_range - p_pivots_sum_1 = 0.0 - min_larger_1 = 1.0 - num_min_larger_1 = tl.zeros((), dtype=tl.uint32) - - # First pass: Calculate p_pivots_sum and min_larger + # Single fused pass: compute p_pivots_sum, + # min_larger, and num_min_larger together. + # See _update_min_larger_stats for the + # tile-level merge logic. for i in range(0, NUM_TILES): offs_n = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) mask_n = offs_n < VOCAB_SIZE @@ -850,51 +777,18 @@ def _topk_topp_kernel( BUFFER_ROW + offs_n, mask=mask_n, other=0.0 ) - p_pivots_sum_0 += tl.sum( - probs_blk * (probs_blk > p_pivot_0) - ) - masked_larger_0 = tl.where( - probs_blk > p_pivot_0, probs_blk, 1.0 - ) - min_larger_0 = tl.minimum( - min_larger_0, tl.min(masked_larger_0) + above_0 = probs_blk > p_pivot_0 + p_pivots_sum_0 += tl.sum(probs_blk * above_0) + + min_larger_0, num_min_larger_0 = _update_min_larger_stats( + probs_blk, + above_0, + min_larger_0, + num_min_larger_0, + 1.0, ) - p_pivots_sum_1 += tl.sum( - probs_blk * (probs_blk > p_pivot_1) - ) - masked_larger_1 = tl.where( - probs_blk > p_pivot_1, probs_blk, 1.0 - ) - min_larger_1 = tl.minimum( - min_larger_1, tl.min(masked_larger_1) - ) - - # Second pass: Calculate num_min_larger - for i in range(0, NUM_TILES): - offs_n = i * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE) - mask_n = offs_n < VOCAB_SIZE - probs_blk = tl.load( - BUFFER_ROW + offs_n, mask=mask_n, other=0.0 - ) - - num_min_larger_0 += tl.sum( - tl.abs(probs_blk - min_larger_0) < 1e-9 - ) - num_min_larger_1 += tl.sum( - tl.abs(probs_blk - min_larger_1) < 1e-9 - ) - - # Check if any of the pivots satisfy termination condition - if ( - p_pivots_sum_1 >= p - and p_pivots_sum_1 - (min_larger_1 * num_min_larger_1) < p - ): - p_pivot = p_pivot_1 - min_larger_prob = min_larger_1 - num_min_larger = num_min_larger_1 - p_pivots_sum = p_pivots_sum_1 - found_pivot = 1 + # Check if the pivot satisfies termination condition if ( p_pivots_sum_0 >= p and p_pivots_sum_0 - (min_larger_0 * num_min_larger_0) < p @@ -906,22 +800,20 @@ def _topk_topp_kernel( found_pivot = 1 # Update range - if p_pivots_sum_1 > p: - min_range = p_pivot_1 - elif p_pivots_sum_0 > p: + if p_pivots_sum_0 > p: min_range = p_pivot_0 - - if p_pivots_sum_0 < p: + elif p_pivots_sum_0 < p: max_range = p_pivot_0 - elif p_pivots_sum_1 < p: - max_range = p_pivot_1 num_iters += 1 if (max_range - min_range) < 1e-9 or num_iters >= 18: p_pivot = (max_range + min_range) / 2.0 + min_larger_prob = min_larger_0 + num_min_larger = num_min_larger_0 + p_pivots_sum = p_pivots_sum_0 found_pivot = 1 - duplicate_logit = tl.log(min_larger_prob * sum_exp_logits) + max_logit + duplicate_logit = tl.log(min_larger_prob * sum_exp_logits) + max_sample num_duplicate_logit = num_min_larger num_keep = num_duplicate_logit - tl.cast( (p_pivots_sum - p) / min_larger_prob, tl.uint32 @@ -952,9 +844,7 @@ def _topk_topp_kernel( tl.abs(logits_blk - duplicate_logit) < 1e-9 ) & mask_n duplicate_count = tl.cumsum(duplicate_mask) + num_kept - duplicate_keep_mask = ( - duplicate_count <= num_duplicate_logit - ) & duplicate_mask + duplicate_keep_mask = (duplicate_count <= num_keep) & duplicate_mask duplicate_remove_mask = duplicate_mask & ~duplicate_keep_mask num_kept += tl.sum(duplicate_keep_mask) keep_mask = keep_mask & (~duplicate_remove_mask)