[Perf][Kernel] Add sampled filtering for persistent top-k (#56346)
Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
co-authored by
OpenAI Codex
parent
1257512305
commit
c6fa1f05d1
@@ -1030,41 +1030,34 @@ constexpr uint32_t FILTERED_TOPK_SMEM_INPUT_SIZE =
|
||||
constexpr size_t FILTERED_TOPK_SMEM_DYNAMIC =
|
||||
sizeof(int) * 2 * FILTERED_TOPK_SMEM_INPUT_SIZE; // 128KB
|
||||
|
||||
/*!
|
||||
* \brief Filtered Top-K kernel for ragged sequences.
|
||||
*
|
||||
* \tparam DType Data type (float, half, nv_bfloat16)
|
||||
* \tparam IdType Index type (int32_t)
|
||||
* \tparam VEC_SIZE Vector size for input loads (1, 2, 4, or 8)
|
||||
*/
|
||||
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K = 2048,
|
||||
bool UsePredicatedShortLoads = false>
|
||||
__global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
|
||||
FilteredTopKUnifiedKernel(const DType* __restrict__ input,
|
||||
IdType* __restrict__ output,
|
||||
const IdType* __restrict__ lengths,
|
||||
uint32_t num_rows, uint32_t top_k,
|
||||
uint32_t max_len) {
|
||||
template <uint32_t MAX_K>
|
||||
struct FilteredTopKStorage {
|
||||
alignas(128) int histogram[2][256 + 128];
|
||||
alignas(128) int counter;
|
||||
alignas(128) int threshold_bin;
|
||||
alignas(128) int num_input[2];
|
||||
alignas(128) int indices[MAX_K];
|
||||
int last_remain;
|
||||
};
|
||||
|
||||
// With CheckOverflow, return false before using a truncated stash so the caller
|
||||
// can retry with exact full-row selection.
|
||||
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K,
|
||||
bool UsePredicatedShortLoads, bool CheckOverflow = false>
|
||||
__device__ __forceinline__ bool filtered_topk_row(
|
||||
const DType* score, IdType* dst, int length, uint32_t top_k,
|
||||
FilteredTopKStorage<MAX_K>& storage) {
|
||||
constexpr uint32_t BLOCK_SIZE = FILTERED_TOPK_BLOCK_THREADS;
|
||||
constexpr int RADIX = 256;
|
||||
constexpr int SMEM_INPUT_SIZE = FILTERED_TOPK_SMEM_INPUT_SIZE;
|
||||
|
||||
const uint32_t bid = blockIdx.x;
|
||||
const int tx = threadIdx.x;
|
||||
|
||||
if (bid >= num_rows) return;
|
||||
|
||||
const int length =
|
||||
(lengths != nullptr) ? lengths[bid] : static_cast<int>(max_len);
|
||||
const DType* score = input + bid * max_len;
|
||||
IdType* dst = output + bid * top_k;
|
||||
|
||||
// Trivial case: length <= top_k
|
||||
if (length <= static_cast<int>(top_k)) {
|
||||
for (int i = tx; i < static_cast<int>(top_k); i += BLOCK_SIZE) {
|
||||
dst[i] = (i < length) ? static_cast<IdType>(i) : static_cast<IdType>(-1);
|
||||
}
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Short path
|
||||
@@ -1077,16 +1070,14 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
|
||||
hist4096::histogram_4096_topk<MAX_K, 12, 8>(score, dst, length,
|
||||
_smem_reg);
|
||||
}
|
||||
return;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Static shared memory
|
||||
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
|
||||
alignas(128) __shared__ int s_counter;
|
||||
alignas(128) __shared__ int s_threshold_bin_id;
|
||||
alignas(128) __shared__ int s_num_input[2];
|
||||
alignas(128) __shared__ int s_indices[MAX_K];
|
||||
|
||||
auto& s_histogram_buf = storage.histogram;
|
||||
auto& s_counter = storage.counter;
|
||||
auto& s_threshold_bin_id = storage.threshold_bin;
|
||||
auto& s_num_input = storage.num_input;
|
||||
auto& s_indices = storage.indices;
|
||||
auto& s_histogram = s_histogram_buf[0];
|
||||
|
||||
// Dynamic shared memory for input double buffer
|
||||
@@ -1213,10 +1204,13 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
|
||||
// Stage 2: refine with 8bit radix passes
|
||||
#pragma unroll
|
||||
for (int round = 0; round < NUM_ROUNDS; ++round) {
|
||||
__shared__ int s_last_remain;
|
||||
auto& s_last_remain = storage.last_remain;
|
||||
const auto r_idx = round % 2;
|
||||
|
||||
const auto _raw_num_input = s_num_input[r_idx];
|
||||
if constexpr (CheckOverflow) {
|
||||
if (_raw_num_input > SMEM_INPUT_SIZE) return false;
|
||||
}
|
||||
const auto num_input =
|
||||
(_raw_num_input < SMEM_INPUT_SIZE) ? _raw_num_input : SMEM_INPUT_SIZE;
|
||||
|
||||
@@ -1284,6 +1278,30 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
|
||||
const int idx = s_indices[base];
|
||||
dst[base] = static_cast<IdType>(idx);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/*!
|
||||
* \brief Filtered Top-K kernel for ragged sequences.
|
||||
*
|
||||
* \tparam DType Data type (float, half, nv_bfloat16)
|
||||
* \tparam IdType Index type (int32_t)
|
||||
* \tparam VEC_SIZE Vector size for input loads (1, 2, 4, or 8)
|
||||
*/
|
||||
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K = 2048,
|
||||
bool UsePredicatedShortLoads = false>
|
||||
__global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
|
||||
FilteredTopKUnifiedKernel(const DType* __restrict__ input,
|
||||
IdType* __restrict__ output,
|
||||
const IdType* __restrict__ lengths,
|
||||
uint32_t num_rows, uint32_t top_k,
|
||||
uint32_t max_len) {
|
||||
const uint32_t bid = blockIdx.x;
|
||||
if (bid >= num_rows) return;
|
||||
const int length = lengths ? lengths[bid] : static_cast<int>(max_len);
|
||||
__shared__ FilteredTopKStorage<MAX_K> storage;
|
||||
filtered_topk_row<DType, IdType, VEC_SIZE, MAX_K, UsePredicatedShortLoads>(
|
||||
input + bid * max_len, output + bid * top_k, length, top_k, storage);
|
||||
}
|
||||
|
||||
// Helper to compute GCD for VEC_SIZE selection
|
||||
|
||||
@@ -0,0 +1,253 @@
|
||||
// SPDX-License-Identifier: Apache-2.0
|
||||
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
// Sampled top-k for long FP32 sparse-indexer rows, inspired by DeepSelect:
|
||||
// https://github.com/deepseek-ai/DeepSelect
|
||||
// A sample estimates a cutoff, then survivors are compacted in shared memory
|
||||
// for exact FP32 selection. Too few or too many candidates trigger an exact
|
||||
// full-row fallback.
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cub/cub.cuh>
|
||||
#include <cuda_runtime.h>
|
||||
#include <cstdint>
|
||||
|
||||
#include "persistent_topk.cuh"
|
||||
|
||||
namespace vllm::sampled_topk {
|
||||
|
||||
constexpr int kThreads = 1024;
|
||||
// Conservative crossover bounds from the B300 batch/length sweep.
|
||||
template <int K>
|
||||
constexpr int kMinSampledLength = K == 512 ? 98304 : 65536;
|
||||
// Target half the buffer to leave room for sampling error.
|
||||
constexpr int kSample = 4096;
|
||||
constexpr int kCapacity = 8192;
|
||||
|
||||
struct Storage {
|
||||
uint2 candidates[kCapacity];
|
||||
int histogram[2048];
|
||||
typename cub::BlockScan<int, kThreads>::TempStorage scan;
|
||||
int count;
|
||||
int remaining;
|
||||
int bin;
|
||||
uint32_t prefix;
|
||||
};
|
||||
|
||||
__device__ __forceinline__ uint32_t ordered(float value) {
|
||||
const uint32_t bits = __float_as_uint(value);
|
||||
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t coarse(uint32_t key) {
|
||||
const uint32_t bits = (key & 0x80000000u) ? (key ^ 0x80000000u) : ~key;
|
||||
return topk_histogram_4096::extract_coarse_bin_N<11>(__uint_as_float(bits));
|
||||
}
|
||||
|
||||
// Select the bin containing the rank and update the rank within that bin.
|
||||
template <int Bins>
|
||||
__device__ void select_bin(Storage& s) {
|
||||
constexpr int kItems = (Bins + kThreads - 1) / kThreads;
|
||||
int sums[kItems];
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kItems; ++j) {
|
||||
const int bin = Bins - 1 - (threadIdx.x * kItems + j);
|
||||
sums[j] = bin >= 0 ? s.histogram[bin] : 0;
|
||||
}
|
||||
cub::BlockScan<int, kThreads>(s.scan).InclusiveSum(sums, sums);
|
||||
const int remaining = s.remaining;
|
||||
__syncthreads();
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kItems; ++j) {
|
||||
const int bin = Bins - 1 - (threadIdx.x * kItems + j);
|
||||
if (bin >= 0 && sums[j] >= remaining &&
|
||||
sums[j] - s.histogram[bin] < remaining) {
|
||||
s.bin = bin;
|
||||
s.remaining = remaining - (sums[j] - s.histogram[bin]);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
}
|
||||
|
||||
// Find a coarse bin, then refine its full FP32 keys one byte at a time.
|
||||
// Read shared candidates normally, or the full row for exact fallback.
|
||||
template <bool Buffered>
|
||||
__device__ uint32_t threshold(Storage& s, const float* row, int count,
|
||||
int rank) {
|
||||
const int tid = threadIdx.x;
|
||||
if (tid == 0) {
|
||||
s.remaining = rank;
|
||||
s.prefix = 0;
|
||||
}
|
||||
for (int i = tid; i < 2048; i += kThreads) s.histogram[i] = 0;
|
||||
__syncthreads();
|
||||
for (int i = tid; i < count; i += kThreads) {
|
||||
const uint32_t key = Buffered ? s.candidates[i].y : ordered(row[i]);
|
||||
atomicAdd(&s.histogram[coarse(key)], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
select_bin<2048>(s);
|
||||
const int coarse_bin = s.bin;
|
||||
uint32_t mask = 0;
|
||||
for (int shift = 24; shift >= 0; shift -= 8) {
|
||||
if (tid < 256) s.histogram[tid] = 0;
|
||||
__syncthreads();
|
||||
const uint32_t prefix = s.prefix;
|
||||
for (int i = tid; i < count; i += kThreads) {
|
||||
const uint32_t key = Buffered ? s.candidates[i].y : ordered(row[i]);
|
||||
if (coarse(key) == coarse_bin && (key & mask) == prefix) {
|
||||
atomicAdd(&s.histogram[(key >> shift) & 255], 1);
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
select_bin<256>(s);
|
||||
if (tid == 0) s.prefix = prefix | (static_cast<uint32_t>(s.bin) << shift);
|
||||
mask |= 255u << shift;
|
||||
__syncthreads();
|
||||
}
|
||||
return s.prefix;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ int reserve(bool hit, int* counter) {
|
||||
const uint32_t mask = __ballot_sync(0xffffffffu, hit);
|
||||
if (mask == 0) return 0;
|
||||
const int lane = threadIdx.x & 31;
|
||||
int base = 0;
|
||||
if (lane == 0) base = atomicAdd(counter, __popc(mask));
|
||||
base = __shfl_sync(0xffffffffu, base, 0);
|
||||
return base + __popc(mask & ((1u << lane) - 1));
|
||||
}
|
||||
|
||||
template <bool Buffered>
|
||||
__device__ void emit(Storage& s, const float* row, int32_t* dst, int count,
|
||||
uint32_t cutoff) {
|
||||
const int tid = threadIdx.x;
|
||||
if (tid == 0) s.count = 0;
|
||||
__syncthreads();
|
||||
for (int base = 0; base < count; base += kThreads) {
|
||||
const int i = base + tid;
|
||||
const uint2 pair = i < count ? (Buffered ? s.candidates[i]
|
||||
: make_uint2(i, ordered(row[i])))
|
||||
: make_uint2(0, 0);
|
||||
bool keep = i < count && pair.y > cutoff;
|
||||
if (i < count && pair.y == cutoff) {
|
||||
keep = atomicSub(&s.remaining, 1) > 0;
|
||||
}
|
||||
const int offset = reserve(keep, &s.count);
|
||||
if (keep) dst[offset] = pair.x;
|
||||
}
|
||||
}
|
||||
|
||||
template <int K>
|
||||
__global__ void __launch_bounds__(kThreads)
|
||||
sampled_topk_kernel(const float* __restrict__ input,
|
||||
const int32_t* __restrict__ lengths,
|
||||
int32_t* __restrict__ output, int64_t stride,
|
||||
int max_length) {
|
||||
extern __shared__ __align__(16) unsigned char smem[];
|
||||
auto& s = *reinterpret_cast<Storage*>(smem);
|
||||
const int tid = threadIdx.x;
|
||||
const int length = max(0, min(lengths[blockIdx.x], max_length));
|
||||
const float* row = input + blockIdx.x * stride;
|
||||
int32_t* dst = output + blockIdx.x * K;
|
||||
// Graph allocations can hold rows much shorter than the host length bound.
|
||||
if (length < kMinSampledLength<K>) {
|
||||
__shared__ filtered_topk::FilteredTopKStorage<K> fallback_storage;
|
||||
bool selected;
|
||||
if ((stride & 3) == 0 && (reinterpret_cast<uintptr_t>(row) & 15) == 0) {
|
||||
selected =
|
||||
filtered_topk::filtered_topk_row<float, int32_t, 4, K, false, true>(
|
||||
row, dst, length, K, fallback_storage);
|
||||
} else {
|
||||
selected =
|
||||
filtered_topk::filtered_topk_row<float, int32_t, 1, K, true, true>(
|
||||
row, dst, length, K, fallback_storage);
|
||||
}
|
||||
if (!selected) {
|
||||
const uint32_t exact_cutoff = threshold<false>(s, row, length, K);
|
||||
emit<false>(s, row, dst, length, exact_cutoff);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (tid == 0) s.remaining = max(1, (kCapacity / 2) * kSample / length);
|
||||
for (int i = tid; i < 2048; i += kThreads) s.histogram[i] = 0;
|
||||
__syncthreads();
|
||||
// Sample contiguous warps spread across the row to keep loads coalesced.
|
||||
for (int i = tid; i < kSample; i += kThreads) {
|
||||
const int index = (i / 32) * (length / (kSample / 32)) + i % 32;
|
||||
atomicAdd(&s.histogram[coarse(ordered(row[index]))], 1);
|
||||
}
|
||||
__syncthreads();
|
||||
select_bin<2048>(s);
|
||||
const uint16_t half_key = s.bin << 5;
|
||||
const uint16_t half_bits =
|
||||
(half_key & 0x8000u) ? (half_key ^ 0x8000u) : ~half_key;
|
||||
const uint32_t cutoff = ordered(__half2float(__ushort_as_half(half_bits)));
|
||||
if (tid == 0) s.count = 0;
|
||||
__syncthreads();
|
||||
constexpr int kItems = 16;
|
||||
// Compact survivors with one reservation per warp per tile. Keep counting
|
||||
// beyond capacity so overflow triggers fallback instead of using a subset.
|
||||
for (int base = 0; base < length; base += kItems * kThreads) {
|
||||
uint32_t keys[kItems];
|
||||
uint32_t hit_mask = 0;
|
||||
if ((reinterpret_cast<uintptr_t>(row) & 15) == 0 &&
|
||||
base + kItems * kThreads <= length) {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kItems / 4; ++j) {
|
||||
const float4 values =
|
||||
reinterpret_cast<const float4*>(row + base)[tid + j * kThreads];
|
||||
keys[4 * j] = ordered(values.x);
|
||||
keys[4 * j + 1] = ordered(values.y);
|
||||
keys[4 * j + 2] = ordered(values.z);
|
||||
keys[4 * j + 3] = ordered(values.w);
|
||||
}
|
||||
} else {
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kItems; ++j) {
|
||||
const int i = base + tid * 4 + (j / 4) * kThreads * 4 + j % 4;
|
||||
keys[j] = i < length ? ordered(row[i]) : 0;
|
||||
}
|
||||
}
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kItems; ++j) {
|
||||
const int i = base + tid * 4 + (j / 4) * kThreads * 4 + j % 4;
|
||||
if (i < length && keys[j] >= cutoff) hit_mask |= 1u << j;
|
||||
}
|
||||
const int lane = tid & 31;
|
||||
const int hits = __popc(hit_mask);
|
||||
int inclusive = hits;
|
||||
#pragma unroll
|
||||
for (int delta = 1; delta < 32; delta *= 2) {
|
||||
const int previous = __shfl_up_sync(0xffffffffu, inclusive, delta);
|
||||
if (lane >= delta) inclusive += previous;
|
||||
}
|
||||
const int total = __shfl_sync(0xffffffffu, inclusive, 31);
|
||||
int warp_base = 0;
|
||||
if (lane == 31 && total != 0) warp_base = atomicAdd(&s.count, total);
|
||||
warp_base = __shfl_sync(0xffffffffu, warp_base, 31);
|
||||
int offset = warp_base + inclusive - hits;
|
||||
#pragma unroll
|
||||
for (int j = 0; j < kItems; ++j) {
|
||||
if (hit_mask & (1u << j)) {
|
||||
if (offset < kCapacity) {
|
||||
s.candidates[offset] = make_uint2(
|
||||
base + tid * 4 + (j / 4) * kThreads * 4 + j % 4, keys[j]);
|
||||
}
|
||||
++offset;
|
||||
}
|
||||
}
|
||||
}
|
||||
__syncthreads();
|
||||
const int count = s.count;
|
||||
if (count >= K && count <= kCapacity) {
|
||||
const uint32_t exact_cutoff = threshold<true>(s, row, count, K);
|
||||
emit<true>(s, row, dst, count, exact_cutoff);
|
||||
} else {
|
||||
const uint32_t exact_cutoff = threshold<false>(s, row, length, K);
|
||||
emit<false>(s, row, dst, length, exact_cutoff);
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace vllm::sampled_topk
|
||||
@@ -9,6 +9,7 @@
|
||||
|
||||
#ifndef USE_ROCM
|
||||
#include "persistent_topk.cuh"
|
||||
#include "sampled_topk.cuh"
|
||||
#endif
|
||||
|
||||
namespace {
|
||||
@@ -36,7 +37,22 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
|
||||
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
|
||||
}
|
||||
|
||||
if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
|
||||
// Allow static fallback storage in addition to the 128 KiB dynamic buffer.
|
||||
if (num_rows > 64 &&
|
||||
max_seq_len >= vllm::sampled_topk::kMinSampledLength<TopK> &&
|
||||
max_smem_per_block >= 144 * 1024) {
|
||||
auto kernel = vllm::sampled_topk::sampled_topk_kernel<TopK>;
|
||||
constexpr size_t smem_size =
|
||||
vllm::filtered_topk::FILTERED_TOPK_SMEM_DYNAMIC;
|
||||
cudaError_t status = cudaFuncSetAttribute(
|
||||
kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
|
||||
STD_TORCH_CHECK(status == cudaSuccess,
|
||||
"sampled_topk smem failed: ", cudaGetErrorString(status));
|
||||
kernel<<<num_rows, vllm::sampled_topk::kThreads, smem_size, stream>>>(
|
||||
logits.const_data_ptr<float>(), lengths.const_data_ptr<int32_t>(),
|
||||
output.mutable_data_ptr<int32_t>(), stride,
|
||||
static_cast<int>(std::min(max_seq_len, logits.size(1))));
|
||||
} else if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
|
||||
cudaError_t status =
|
||||
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
|
||||
logits.const_data_ptr<float>(), output.mutable_data_ptr<int32_t>(),
|
||||
|
||||
Reference in New Issue
Block a user