[Perf][Kernel] Add sampled filtering for persistent top-k (#56346)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Misha Goin
2026-09-15 14:15:59 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 1257512305
commit c6fa1f05d1
5 changed files with 556 additions and 35 deletions
+52 -34
View File
@@ -1030,41 +1030,34 @@ constexpr uint32_t FILTERED_TOPK_SMEM_INPUT_SIZE =
constexpr size_t FILTERED_TOPK_SMEM_DYNAMIC =
sizeof(int) * 2 * FILTERED_TOPK_SMEM_INPUT_SIZE; // 128KB
/*!
* \brief Filtered Top-K kernel for ragged sequences.
*
* \tparam DType Data type (float, half, nv_bfloat16)
* \tparam IdType Index type (int32_t)
* \tparam VEC_SIZE Vector size for input loads (1, 2, 4, or 8)
*/
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K = 2048,
bool UsePredicatedShortLoads = false>
__global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
FilteredTopKUnifiedKernel(const DType* __restrict__ input,
IdType* __restrict__ output,
const IdType* __restrict__ lengths,
uint32_t num_rows, uint32_t top_k,
uint32_t max_len) {
template <uint32_t MAX_K>
struct FilteredTopKStorage {
alignas(128) int histogram[2][256 + 128];
alignas(128) int counter;
alignas(128) int threshold_bin;
alignas(128) int num_input[2];
alignas(128) int indices[MAX_K];
int last_remain;
};
// With CheckOverflow, return false before using a truncated stash so the caller
// can retry with exact full-row selection.
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K,
bool UsePredicatedShortLoads, bool CheckOverflow = false>
__device__ __forceinline__ bool filtered_topk_row(
const DType* score, IdType* dst, int length, uint32_t top_k,
FilteredTopKStorage<MAX_K>& storage) {
constexpr uint32_t BLOCK_SIZE = FILTERED_TOPK_BLOCK_THREADS;
constexpr int RADIX = 256;
constexpr int SMEM_INPUT_SIZE = FILTERED_TOPK_SMEM_INPUT_SIZE;
const uint32_t bid = blockIdx.x;
const int tx = threadIdx.x;
if (bid >= num_rows) return;
const int length =
(lengths != nullptr) ? lengths[bid] : static_cast<int>(max_len);
const DType* score = input + bid * max_len;
IdType* dst = output + bid * top_k;
// Trivial case: length <= top_k
if (length <= static_cast<int>(top_k)) {
for (int i = tx; i < static_cast<int>(top_k); i += BLOCK_SIZE) {
dst[i] = (i < length) ? static_cast<IdType>(i) : static_cast<IdType>(-1);
}
return;
return true;
}
// Short path
@@ -1077,16 +1070,14 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
hist4096::histogram_4096_topk<MAX_K, 12, 8>(score, dst, length,
_smem_reg);
}
return;
return true;
}
// Static shared memory
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
alignas(128) __shared__ int s_counter;
alignas(128) __shared__ int s_threshold_bin_id;
alignas(128) __shared__ int s_num_input[2];
alignas(128) __shared__ int s_indices[MAX_K];
auto& s_histogram_buf = storage.histogram;
auto& s_counter = storage.counter;
auto& s_threshold_bin_id = storage.threshold_bin;
auto& s_num_input = storage.num_input;
auto& s_indices = storage.indices;
auto& s_histogram = s_histogram_buf[0];
// Dynamic shared memory for input double buffer
@@ -1213,10 +1204,13 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
// Stage 2: refine with 8bit radix passes
#pragma unroll
for (int round = 0; round < NUM_ROUNDS; ++round) {
__shared__ int s_last_remain;
auto& s_last_remain = storage.last_remain;
const auto r_idx = round % 2;
const auto _raw_num_input = s_num_input[r_idx];
if constexpr (CheckOverflow) {
if (_raw_num_input > SMEM_INPUT_SIZE) return false;
}
const auto num_input =
(_raw_num_input < SMEM_INPUT_SIZE) ? _raw_num_input : SMEM_INPUT_SIZE;
@@ -1284,6 +1278,30 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
const int idx = s_indices[base];
dst[base] = static_cast<IdType>(idx);
}
return true;
}
/*!
* \brief Filtered Top-K kernel for ragged sequences.
*
* \tparam DType Data type (float, half, nv_bfloat16)
* \tparam IdType Index type (int32_t)
* \tparam VEC_SIZE Vector size for input loads (1, 2, 4, or 8)
*/
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K = 2048,
bool UsePredicatedShortLoads = false>
__global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
FilteredTopKUnifiedKernel(const DType* __restrict__ input,
IdType* __restrict__ output,
const IdType* __restrict__ lengths,
uint32_t num_rows, uint32_t top_k,
uint32_t max_len) {
const uint32_t bid = blockIdx.x;
if (bid >= num_rows) return;
const int length = lengths ? lengths[bid] : static_cast<int>(max_len);
__shared__ FilteredTopKStorage<MAX_K> storage;
filtered_topk_row<DType, IdType, VEC_SIZE, MAX_K, UsePredicatedShortLoads>(
input + bid * max_len, output + bid * top_k, length, top_k, storage);
}
// Helper to compute GCD for VEC_SIZE selection
+253
View File
@@ -0,0 +1,253 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// Sampled top-k for long FP32 sparse-indexer rows, inspired by DeepSelect:
// https://github.com/deepseek-ai/DeepSelect
// A sample estimates a cutoff, then survivors are compacted in shared memory
// for exact FP32 selection. Too few or too many candidates trigger an exact
// full-row fallback.
#pragma once
#include <cub/cub.cuh>
#include <cuda_runtime.h>
#include <cstdint>
#include "persistent_topk.cuh"
namespace vllm::sampled_topk {
constexpr int kThreads = 1024;
// Conservative crossover bounds from the B300 batch/length sweep.
template <int K>
constexpr int kMinSampledLength = K == 512 ? 98304 : 65536;
// Target half the buffer to leave room for sampling error.
constexpr int kSample = 4096;
constexpr int kCapacity = 8192;
struct Storage {
uint2 candidates[kCapacity];
int histogram[2048];
typename cub::BlockScan<int, kThreads>::TempStorage scan;
int count;
int remaining;
int bin;
uint32_t prefix;
};
__device__ __forceinline__ uint32_t ordered(float value) {
const uint32_t bits = __float_as_uint(value);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ __forceinline__ uint32_t coarse(uint32_t key) {
const uint32_t bits = (key & 0x80000000u) ? (key ^ 0x80000000u) : ~key;
return topk_histogram_4096::extract_coarse_bin_N<11>(__uint_as_float(bits));
}
// Select the bin containing the rank and update the rank within that bin.
template <int Bins>
__device__ void select_bin(Storage& s) {
constexpr int kItems = (Bins + kThreads - 1) / kThreads;
int sums[kItems];
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int bin = Bins - 1 - (threadIdx.x * kItems + j);
sums[j] = bin >= 0 ? s.histogram[bin] : 0;
}
cub::BlockScan<int, kThreads>(s.scan).InclusiveSum(sums, sums);
const int remaining = s.remaining;
__syncthreads();
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int bin = Bins - 1 - (threadIdx.x * kItems + j);
if (bin >= 0 && sums[j] >= remaining &&
sums[j] - s.histogram[bin] < remaining) {
s.bin = bin;
s.remaining = remaining - (sums[j] - s.histogram[bin]);
}
}
__syncthreads();
}
// Find a coarse bin, then refine its full FP32 keys one byte at a time.
// Read shared candidates normally, or the full row for exact fallback.
template <bool Buffered>
__device__ uint32_t threshold(Storage& s, const float* row, int count,
int rank) {
const int tid = threadIdx.x;
if (tid == 0) {
s.remaining = rank;
s.prefix = 0;
}
for (int i = tid; i < 2048; i += kThreads) s.histogram[i] = 0;
__syncthreads();
for (int i = tid; i < count; i += kThreads) {
const uint32_t key = Buffered ? s.candidates[i].y : ordered(row[i]);
atomicAdd(&s.histogram[coarse(key)], 1);
}
__syncthreads();
select_bin<2048>(s);
const int coarse_bin = s.bin;
uint32_t mask = 0;
for (int shift = 24; shift >= 0; shift -= 8) {
if (tid < 256) s.histogram[tid] = 0;
__syncthreads();
const uint32_t prefix = s.prefix;
for (int i = tid; i < count; i += kThreads) {
const uint32_t key = Buffered ? s.candidates[i].y : ordered(row[i]);
if (coarse(key) == coarse_bin && (key & mask) == prefix) {
atomicAdd(&s.histogram[(key >> shift) & 255], 1);
}
}
__syncthreads();
select_bin<256>(s);
if (tid == 0) s.prefix = prefix | (static_cast<uint32_t>(s.bin) << shift);
mask |= 255u << shift;
__syncthreads();
}
return s.prefix;
}
__device__ __forceinline__ int reserve(bool hit, int* counter) {
const uint32_t mask = __ballot_sync(0xffffffffu, hit);
if (mask == 0) return 0;
const int lane = threadIdx.x & 31;
int base = 0;
if (lane == 0) base = atomicAdd(counter, __popc(mask));
base = __shfl_sync(0xffffffffu, base, 0);
return base + __popc(mask & ((1u << lane) - 1));
}
template <bool Buffered>
__device__ void emit(Storage& s, const float* row, int32_t* dst, int count,
uint32_t cutoff) {
const int tid = threadIdx.x;
if (tid == 0) s.count = 0;
__syncthreads();
for (int base = 0; base < count; base += kThreads) {
const int i = base + tid;
const uint2 pair = i < count ? (Buffered ? s.candidates[i]
: make_uint2(i, ordered(row[i])))
: make_uint2(0, 0);
bool keep = i < count && pair.y > cutoff;
if (i < count && pair.y == cutoff) {
keep = atomicSub(&s.remaining, 1) > 0;
}
const int offset = reserve(keep, &s.count);
if (keep) dst[offset] = pair.x;
}
}
template <int K>
__global__ void __launch_bounds__(kThreads)
sampled_topk_kernel(const float* __restrict__ input,
const int32_t* __restrict__ lengths,
int32_t* __restrict__ output, int64_t stride,
int max_length) {
extern __shared__ __align__(16) unsigned char smem[];
auto& s = *reinterpret_cast<Storage*>(smem);
const int tid = threadIdx.x;
const int length = max(0, min(lengths[blockIdx.x], max_length));
const float* row = input + blockIdx.x * stride;
int32_t* dst = output + blockIdx.x * K;
// Graph allocations can hold rows much shorter than the host length bound.
if (length < kMinSampledLength<K>) {
__shared__ filtered_topk::FilteredTopKStorage<K> fallback_storage;
bool selected;
if ((stride & 3) == 0 && (reinterpret_cast<uintptr_t>(row) & 15) == 0) {
selected =
filtered_topk::filtered_topk_row<float, int32_t, 4, K, false, true>(
row, dst, length, K, fallback_storage);
} else {
selected =
filtered_topk::filtered_topk_row<float, int32_t, 1, K, true, true>(
row, dst, length, K, fallback_storage);
}
if (!selected) {
const uint32_t exact_cutoff = threshold<false>(s, row, length, K);
emit<false>(s, row, dst, length, exact_cutoff);
}
return;
}
if (tid == 0) s.remaining = max(1, (kCapacity / 2) * kSample / length);
for (int i = tid; i < 2048; i += kThreads) s.histogram[i] = 0;
__syncthreads();
// Sample contiguous warps spread across the row to keep loads coalesced.
for (int i = tid; i < kSample; i += kThreads) {
const int index = (i / 32) * (length / (kSample / 32)) + i % 32;
atomicAdd(&s.histogram[coarse(ordered(row[index]))], 1);
}
__syncthreads();
select_bin<2048>(s);
const uint16_t half_key = s.bin << 5;
const uint16_t half_bits =
(half_key & 0x8000u) ? (half_key ^ 0x8000u) : ~half_key;
const uint32_t cutoff = ordered(__half2float(__ushort_as_half(half_bits)));
if (tid == 0) s.count = 0;
__syncthreads();
constexpr int kItems = 16;
// Compact survivors with one reservation per warp per tile. Keep counting
// beyond capacity so overflow triggers fallback instead of using a subset.
for (int base = 0; base < length; base += kItems * kThreads) {
uint32_t keys[kItems];
uint32_t hit_mask = 0;
if ((reinterpret_cast<uintptr_t>(row) & 15) == 0 &&
base + kItems * kThreads <= length) {
#pragma unroll
for (int j = 0; j < kItems / 4; ++j) {
const float4 values =
reinterpret_cast<const float4*>(row + base)[tid + j * kThreads];
keys[4 * j] = ordered(values.x);
keys[4 * j + 1] = ordered(values.y);
keys[4 * j + 2] = ordered(values.z);
keys[4 * j + 3] = ordered(values.w);
}
} else {
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int i = base + tid * 4 + (j / 4) * kThreads * 4 + j % 4;
keys[j] = i < length ? ordered(row[i]) : 0;
}
}
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int i = base + tid * 4 + (j / 4) * kThreads * 4 + j % 4;
if (i < length && keys[j] >= cutoff) hit_mask |= 1u << j;
}
const int lane = tid & 31;
const int hits = __popc(hit_mask);
int inclusive = hits;
#pragma unroll
for (int delta = 1; delta < 32; delta *= 2) {
const int previous = __shfl_up_sync(0xffffffffu, inclusive, delta);
if (lane >= delta) inclusive += previous;
}
const int total = __shfl_sync(0xffffffffu, inclusive, 31);
int warp_base = 0;
if (lane == 31 && total != 0) warp_base = atomicAdd(&s.count, total);
warp_base = __shfl_sync(0xffffffffu, warp_base, 31);
int offset = warp_base + inclusive - hits;
#pragma unroll
for (int j = 0; j < kItems; ++j) {
if (hit_mask & (1u << j)) {
if (offset < kCapacity) {
s.candidates[offset] = make_uint2(
base + tid * 4 + (j / 4) * kThreads * 4 + j % 4, keys[j]);
}
++offset;
}
}
}
__syncthreads();
const int count = s.count;
if (count >= K && count <= kCapacity) {
const uint32_t exact_cutoff = threshold<true>(s, row, count, K);
emit<true>(s, row, dst, count, exact_cutoff);
} else {
const uint32_t exact_cutoff = threshold<false>(s, row, length, K);
emit<false>(s, row, dst, length, exact_cutoff);
}
}
} // namespace vllm::sampled_topk
+17 -1
View File
@@ -9,6 +9,7 @@
#ifndef USE_ROCM
#include "persistent_topk.cuh"
#include "sampled_topk.cuh"
#endif
namespace {
@@ -36,7 +37,22 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
}
if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
// Allow static fallback storage in addition to the 128 KiB dynamic buffer.
if (num_rows > 64 &&
max_seq_len >= vllm::sampled_topk::kMinSampledLength<TopK> &&
max_smem_per_block >= 144 * 1024) {
auto kernel = vllm::sampled_topk::sampled_topk_kernel<TopK>;
constexpr size_t smem_size =
vllm::filtered_topk::FILTERED_TOPK_SMEM_DYNAMIC;
cudaError_t status = cudaFuncSetAttribute(
kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
STD_TORCH_CHECK(status == cudaSuccess,
"sampled_topk smem failed: ", cudaGetErrorString(status));
kernel<<<num_rows, vllm::sampled_topk::kThreads, smem_size, stream>>>(
logits.const_data_ptr<float>(), lengths.const_data_ptr<int32_t>(),
output.mutable_data_ptr<int32_t>(), stride,
static_cast<int>(std::min(max_seq_len, logits.size(1))));
} else if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
cudaError_t status =
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
logits.const_data_ptr<float>(), output.mutable_data_ptr<int32_t>(),