[Perf][Kernel] Add sampled filtering for persistent top-k (#56346)

Signed-off-by: mgoin <mgoin64@gmail.com>
Co-authored-by: OpenAI Codex <noreply@openai.com>
This commit is contained in:
Misha Goin
2026-09-15 14:15:59 -07:00
committed by GitHub
co-authored by OpenAI Codex
parent 1257512305
commit c6fa1f05d1
5 changed files with 556 additions and 35 deletions
@@ -0,0 +1,134 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
"""Benchmark sparse-indexer decode top-k on variable-length FP32 rows.
Run on a reserved GPU, for example:
chg run -- .venv/bin/python benchmarks/kernels/benchmark_persistent_topk.py
Use --capacity to emulate graphs captured at a larger maximum context length.
Run the same command on the base and candidate builds to compare JSON results.
"""
import argparse
import itertools
import json
import statistics
from pathlib import Path
import torch
from flashinfer.testing import bench_gpu_time_with_cupti
import vllm._custom_ops # noqa: F401
from vllm.platforms import current_platform
def benchmark(rows: int, width: int, k: int, args) -> dict:
capacity = args.capacity or width
if capacity < width:
raise ValueError("Capacity must be at least the valid-length upper bound")
torch.manual_seed(args.seed)
logits = torch.randn(rows, capacity, device="cuda", dtype=torch.float32)
if args.full_length:
lengths = torch.full((rows,), width, device="cuda", dtype=torch.int32)
else:
lengths = torch.randint(
int(width * 0.8), width + 1, (rows,), device="cuda", dtype=torch.int32
)
invalid = torch.arange(capacity, device="cuda")[None] >= lengths[:, None]
logits.masked_fill_(invalid, float("nan"))
out = torch.empty((rows, k), device="cuda", dtype=torch.int32)
workspace = torch.empty(1024 * 1024, device="cuda", dtype=torch.uint8)
cooperative = (
args.backend == "auto"
and rows <= 64
and capacity % 4 == 0
and current_platform.has_device_capability(90)
and not current_platform.is_device_capability_family(120)
)
op = torch.ops._C.cooperative_topk if cooperative else torch.ops._C.persistent_topk
def run():
op(logits, lengths, out, workspace, k, capacity)
run()
valid = torch.arange(k, device="cuda")[None] < lengths[:, None]
assert torch.all(out[~valid] == -1)
assert torch.all(((out >= 0) & (out < lengths[:, None])) == valid)
sorted_indices = out.sort(dim=1).values
assert torch.all(
(sorted_indices[:, 1:] != sorted_indices[:, :-1])
| (sorted_indices[:, 1:] == -1)
)
selected = logits.gather(1, out.clamp_min(0).long()).masked_fill(
~valid, -float("inf")
)
reference = logits.masked_fill(invalid, -float("inf")).topk(k, dim=1).values
torch.testing.assert_close(
selected.sort(dim=1, descending=True).values, reference, atol=0, rtol=0
)
trials = [
statistics.median(
bench_gpu_time_with_cupti(
run,
use_cuda_graph=True,
cold_l2_cache=not args.warm,
dry_run_iters=5,
repeat_iters=args.repeat,
)
)
* 1000
for _ in range(args.trials)
]
us = statistics.median(trials)
return dict(
rows=rows,
width=width,
capacity=capacity,
k=k,
backend="cooperative" if cooperative else "persistent",
us=us,
trials_us=trials,
# Minimum useful traffic: one valid-score read and one index write.
effective_gbps=(int(lengths.sum()) * 4 + out.numel() * 4) / (us * 1000),
)
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--rows", type=int, nargs="+", default=[1, 32, 128, 256, 1024])
parser.add_argument("--widths", type=int, nargs="+", default=[8192, 65536, 163840])
parser.add_argument("--top-k", type=int, nargs="+", default=[512, 2048])
parser.add_argument("--capacity", type=int)
parser.add_argument(
"--full-length", action="store_true", help="Use exactly --widths valid scores"
)
parser.add_argument("--backend", choices=["auto", "persistent"], default="auto")
parser.add_argument(
"--warm", action="store_true", help="Keep L2 warm between replays"
)
parser.add_argument("--repeat", type=int, default=60)
parser.add_argument("--trials", type=int, default=3)
parser.add_argument("--seed", type=int, default=123)
parser.add_argument("--output", type=Path)
args = parser.parse_args()
metadata = dict(
gpu=str(torch.cuda.get_device_properties(0)),
torch=torch.__version__,
cuda=torch.version.cuda,
args=vars(args),
)
print(json.dumps(metadata, default=str))
results = []
for rows, width, k in itertools.product(args.rows, args.widths, args.top_k):
result = benchmark(rows, width, k, args)
results.append(result)
print(json.dumps(result), flush=True)
if args.output:
args.output.write_text(
json.dumps(dict(metadata=metadata, results=results), indent=2, default=str)
+ "\n"
)
if __name__ == "__main__":
main()
+52 -34
View File
@@ -1030,41 +1030,34 @@ constexpr uint32_t FILTERED_TOPK_SMEM_INPUT_SIZE =
constexpr size_t FILTERED_TOPK_SMEM_DYNAMIC =
sizeof(int) * 2 * FILTERED_TOPK_SMEM_INPUT_SIZE; // 128KB
/*!
* \brief Filtered Top-K kernel for ragged sequences.
*
* \tparam DType Data type (float, half, nv_bfloat16)
* \tparam IdType Index type (int32_t)
* \tparam VEC_SIZE Vector size for input loads (1, 2, 4, or 8)
*/
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K = 2048,
bool UsePredicatedShortLoads = false>
__global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
FilteredTopKUnifiedKernel(const DType* __restrict__ input,
IdType* __restrict__ output,
const IdType* __restrict__ lengths,
uint32_t num_rows, uint32_t top_k,
uint32_t max_len) {
template <uint32_t MAX_K>
struct FilteredTopKStorage {
alignas(128) int histogram[2][256 + 128];
alignas(128) int counter;
alignas(128) int threshold_bin;
alignas(128) int num_input[2];
alignas(128) int indices[MAX_K];
int last_remain;
};
// With CheckOverflow, return false before using a truncated stash so the caller
// can retry with exact full-row selection.
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K,
bool UsePredicatedShortLoads, bool CheckOverflow = false>
__device__ __forceinline__ bool filtered_topk_row(
const DType* score, IdType* dst, int length, uint32_t top_k,
FilteredTopKStorage<MAX_K>& storage) {
constexpr uint32_t BLOCK_SIZE = FILTERED_TOPK_BLOCK_THREADS;
constexpr int RADIX = 256;
constexpr int SMEM_INPUT_SIZE = FILTERED_TOPK_SMEM_INPUT_SIZE;
const uint32_t bid = blockIdx.x;
const int tx = threadIdx.x;
if (bid >= num_rows) return;
const int length =
(lengths != nullptr) ? lengths[bid] : static_cast<int>(max_len);
const DType* score = input + bid * max_len;
IdType* dst = output + bid * top_k;
// Trivial case: length <= top_k
if (length <= static_cast<int>(top_k)) {
for (int i = tx; i < static_cast<int>(top_k); i += BLOCK_SIZE) {
dst[i] = (i < length) ? static_cast<IdType>(i) : static_cast<IdType>(-1);
}
return;
return true;
}
// Short path
@@ -1077,16 +1070,14 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
hist4096::histogram_4096_topk<MAX_K, 12, 8>(score, dst, length,
_smem_reg);
}
return;
return true;
}
// Static shared memory
alignas(128) __shared__ int s_histogram_buf[2][RADIX + 128];
alignas(128) __shared__ int s_counter;
alignas(128) __shared__ int s_threshold_bin_id;
alignas(128) __shared__ int s_num_input[2];
alignas(128) __shared__ int s_indices[MAX_K];
auto& s_histogram_buf = storage.histogram;
auto& s_counter = storage.counter;
auto& s_threshold_bin_id = storage.threshold_bin;
auto& s_num_input = storage.num_input;
auto& s_indices = storage.indices;
auto& s_histogram = s_histogram_buf[0];
// Dynamic shared memory for input double buffer
@@ -1213,10 +1204,13 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
// Stage 2: refine with 8bit radix passes
#pragma unroll
for (int round = 0; round < NUM_ROUNDS; ++round) {
__shared__ int s_last_remain;
auto& s_last_remain = storage.last_remain;
const auto r_idx = round % 2;
const auto _raw_num_input = s_num_input[r_idx];
if constexpr (CheckOverflow) {
if (_raw_num_input > SMEM_INPUT_SIZE) return false;
}
const auto num_input =
(_raw_num_input < SMEM_INPUT_SIZE) ? _raw_num_input : SMEM_INPUT_SIZE;
@@ -1284,6 +1278,30 @@ __global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
const int idx = s_indices[base];
dst[base] = static_cast<IdType>(idx);
}
return true;
}
/*!
* \brief Filtered Top-K kernel for ragged sequences.
*
* \tparam DType Data type (float, half, nv_bfloat16)
* \tparam IdType Index type (int32_t)
* \tparam VEC_SIZE Vector size for input loads (1, 2, 4, or 8)
*/
template <typename DType, typename IdType, int VEC_SIZE, uint32_t MAX_K = 2048,
bool UsePredicatedShortLoads = false>
__global__ void __launch_bounds__(FILTERED_TOPK_BLOCK_THREADS)
FilteredTopKUnifiedKernel(const DType* __restrict__ input,
IdType* __restrict__ output,
const IdType* __restrict__ lengths,
uint32_t num_rows, uint32_t top_k,
uint32_t max_len) {
const uint32_t bid = blockIdx.x;
if (bid >= num_rows) return;
const int length = lengths ? lengths[bid] : static_cast<int>(max_len);
__shared__ FilteredTopKStorage<MAX_K> storage;
filtered_topk_row<DType, IdType, VEC_SIZE, MAX_K, UsePredicatedShortLoads>(
input + bid * max_len, output + bid * top_k, length, top_k, storage);
}
// Helper to compute GCD for VEC_SIZE selection
+253
View File
@@ -0,0 +1,253 @@
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: Copyright contributors to the vLLM project
// Sampled top-k for long FP32 sparse-indexer rows, inspired by DeepSelect:
// https://github.com/deepseek-ai/DeepSelect
// A sample estimates a cutoff, then survivors are compacted in shared memory
// for exact FP32 selection. Too few or too many candidates trigger an exact
// full-row fallback.
#pragma once
#include <cub/cub.cuh>
#include <cuda_runtime.h>
#include <cstdint>
#include "persistent_topk.cuh"
namespace vllm::sampled_topk {
constexpr int kThreads = 1024;
// Conservative crossover bounds from the B300 batch/length sweep.
template <int K>
constexpr int kMinSampledLength = K == 512 ? 98304 : 65536;
// Target half the buffer to leave room for sampling error.
constexpr int kSample = 4096;
constexpr int kCapacity = 8192;
struct Storage {
uint2 candidates[kCapacity];
int histogram[2048];
typename cub::BlockScan<int, kThreads>::TempStorage scan;
int count;
int remaining;
int bin;
uint32_t prefix;
};
__device__ __forceinline__ uint32_t ordered(float value) {
const uint32_t bits = __float_as_uint(value);
return (bits & 0x80000000u) ? ~bits : (bits | 0x80000000u);
}
__device__ __forceinline__ uint32_t coarse(uint32_t key) {
const uint32_t bits = (key & 0x80000000u) ? (key ^ 0x80000000u) : ~key;
return topk_histogram_4096::extract_coarse_bin_N<11>(__uint_as_float(bits));
}
// Select the bin containing the rank and update the rank within that bin.
template <int Bins>
__device__ void select_bin(Storage& s) {
constexpr int kItems = (Bins + kThreads - 1) / kThreads;
int sums[kItems];
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int bin = Bins - 1 - (threadIdx.x * kItems + j);
sums[j] = bin >= 0 ? s.histogram[bin] : 0;
}
cub::BlockScan<int, kThreads>(s.scan).InclusiveSum(sums, sums);
const int remaining = s.remaining;
__syncthreads();
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int bin = Bins - 1 - (threadIdx.x * kItems + j);
if (bin >= 0 && sums[j] >= remaining &&
sums[j] - s.histogram[bin] < remaining) {
s.bin = bin;
s.remaining = remaining - (sums[j] - s.histogram[bin]);
}
}
__syncthreads();
}
// Find a coarse bin, then refine its full FP32 keys one byte at a time.
// Read shared candidates normally, or the full row for exact fallback.
template <bool Buffered>
__device__ uint32_t threshold(Storage& s, const float* row, int count,
int rank) {
const int tid = threadIdx.x;
if (tid == 0) {
s.remaining = rank;
s.prefix = 0;
}
for (int i = tid; i < 2048; i += kThreads) s.histogram[i] = 0;
__syncthreads();
for (int i = tid; i < count; i += kThreads) {
const uint32_t key = Buffered ? s.candidates[i].y : ordered(row[i]);
atomicAdd(&s.histogram[coarse(key)], 1);
}
__syncthreads();
select_bin<2048>(s);
const int coarse_bin = s.bin;
uint32_t mask = 0;
for (int shift = 24; shift >= 0; shift -= 8) {
if (tid < 256) s.histogram[tid] = 0;
__syncthreads();
const uint32_t prefix = s.prefix;
for (int i = tid; i < count; i += kThreads) {
const uint32_t key = Buffered ? s.candidates[i].y : ordered(row[i]);
if (coarse(key) == coarse_bin && (key & mask) == prefix) {
atomicAdd(&s.histogram[(key >> shift) & 255], 1);
}
}
__syncthreads();
select_bin<256>(s);
if (tid == 0) s.prefix = prefix | (static_cast<uint32_t>(s.bin) << shift);
mask |= 255u << shift;
__syncthreads();
}
return s.prefix;
}
__device__ __forceinline__ int reserve(bool hit, int* counter) {
const uint32_t mask = __ballot_sync(0xffffffffu, hit);
if (mask == 0) return 0;
const int lane = threadIdx.x & 31;
int base = 0;
if (lane == 0) base = atomicAdd(counter, __popc(mask));
base = __shfl_sync(0xffffffffu, base, 0);
return base + __popc(mask & ((1u << lane) - 1));
}
template <bool Buffered>
__device__ void emit(Storage& s, const float* row, int32_t* dst, int count,
uint32_t cutoff) {
const int tid = threadIdx.x;
if (tid == 0) s.count = 0;
__syncthreads();
for (int base = 0; base < count; base += kThreads) {
const int i = base + tid;
const uint2 pair = i < count ? (Buffered ? s.candidates[i]
: make_uint2(i, ordered(row[i])))
: make_uint2(0, 0);
bool keep = i < count && pair.y > cutoff;
if (i < count && pair.y == cutoff) {
keep = atomicSub(&s.remaining, 1) > 0;
}
const int offset = reserve(keep, &s.count);
if (keep) dst[offset] = pair.x;
}
}
template <int K>
__global__ void __launch_bounds__(kThreads)
sampled_topk_kernel(const float* __restrict__ input,
const int32_t* __restrict__ lengths,
int32_t* __restrict__ output, int64_t stride,
int max_length) {
extern __shared__ __align__(16) unsigned char smem[];
auto& s = *reinterpret_cast<Storage*>(smem);
const int tid = threadIdx.x;
const int length = max(0, min(lengths[blockIdx.x], max_length));
const float* row = input + blockIdx.x * stride;
int32_t* dst = output + blockIdx.x * K;
// Graph allocations can hold rows much shorter than the host length bound.
if (length < kMinSampledLength<K>) {
__shared__ filtered_topk::FilteredTopKStorage<K> fallback_storage;
bool selected;
if ((stride & 3) == 0 && (reinterpret_cast<uintptr_t>(row) & 15) == 0) {
selected =
filtered_topk::filtered_topk_row<float, int32_t, 4, K, false, true>(
row, dst, length, K, fallback_storage);
} else {
selected =
filtered_topk::filtered_topk_row<float, int32_t, 1, K, true, true>(
row, dst, length, K, fallback_storage);
}
if (!selected) {
const uint32_t exact_cutoff = threshold<false>(s, row, length, K);
emit<false>(s, row, dst, length, exact_cutoff);
}
return;
}
if (tid == 0) s.remaining = max(1, (kCapacity / 2) * kSample / length);
for (int i = tid; i < 2048; i += kThreads) s.histogram[i] = 0;
__syncthreads();
// Sample contiguous warps spread across the row to keep loads coalesced.
for (int i = tid; i < kSample; i += kThreads) {
const int index = (i / 32) * (length / (kSample / 32)) + i % 32;
atomicAdd(&s.histogram[coarse(ordered(row[index]))], 1);
}
__syncthreads();
select_bin<2048>(s);
const uint16_t half_key = s.bin << 5;
const uint16_t half_bits =
(half_key & 0x8000u) ? (half_key ^ 0x8000u) : ~half_key;
const uint32_t cutoff = ordered(__half2float(__ushort_as_half(half_bits)));
if (tid == 0) s.count = 0;
__syncthreads();
constexpr int kItems = 16;
// Compact survivors with one reservation per warp per tile. Keep counting
// beyond capacity so overflow triggers fallback instead of using a subset.
for (int base = 0; base < length; base += kItems * kThreads) {
uint32_t keys[kItems];
uint32_t hit_mask = 0;
if ((reinterpret_cast<uintptr_t>(row) & 15) == 0 &&
base + kItems * kThreads <= length) {
#pragma unroll
for (int j = 0; j < kItems / 4; ++j) {
const float4 values =
reinterpret_cast<const float4*>(row + base)[tid + j * kThreads];
keys[4 * j] = ordered(values.x);
keys[4 * j + 1] = ordered(values.y);
keys[4 * j + 2] = ordered(values.z);
keys[4 * j + 3] = ordered(values.w);
}
} else {
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int i = base + tid * 4 + (j / 4) * kThreads * 4 + j % 4;
keys[j] = i < length ? ordered(row[i]) : 0;
}
}
#pragma unroll
for (int j = 0; j < kItems; ++j) {
const int i = base + tid * 4 + (j / 4) * kThreads * 4 + j % 4;
if (i < length && keys[j] >= cutoff) hit_mask |= 1u << j;
}
const int lane = tid & 31;
const int hits = __popc(hit_mask);
int inclusive = hits;
#pragma unroll
for (int delta = 1; delta < 32; delta *= 2) {
const int previous = __shfl_up_sync(0xffffffffu, inclusive, delta);
if (lane >= delta) inclusive += previous;
}
const int total = __shfl_sync(0xffffffffu, inclusive, 31);
int warp_base = 0;
if (lane == 31 && total != 0) warp_base = atomicAdd(&s.count, total);
warp_base = __shfl_sync(0xffffffffu, warp_base, 31);
int offset = warp_base + inclusive - hits;
#pragma unroll
for (int j = 0; j < kItems; ++j) {
if (hit_mask & (1u << j)) {
if (offset < kCapacity) {
s.candidates[offset] = make_uint2(
base + tid * 4 + (j / 4) * kThreads * 4 + j % 4, keys[j]);
}
++offset;
}
}
}
__syncthreads();
const int count = s.count;
if (count >= K && count <= kCapacity) {
const uint32_t exact_cutoff = threshold<true>(s, row, count, K);
emit<true>(s, row, dst, count, exact_cutoff);
} else {
const uint32_t exact_cutoff = threshold<false>(s, row, length, K);
emit<false>(s, row, dst, length, exact_cutoff);
}
}
} // namespace vllm::sampled_topk
+17 -1
View File
@@ -9,6 +9,7 @@
#ifndef USE_ROCM
#include "persistent_topk.cuh"
#include "sampled_topk.cuh"
#endif
namespace {
@@ -36,7 +37,22 @@ void launch_persistent_topk(const torch::stable::Tensor& logits,
max_smem_per_block = device_prop->sharedMemPerBlockOptin;
}
if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
// Allow static fallback storage in addition to the 128 KiB dynamic buffer.
if (num_rows > 64 &&
max_seq_len >= vllm::sampled_topk::kMinSampledLength<TopK> &&
max_smem_per_block >= 144 * 1024) {
auto kernel = vllm::sampled_topk::sampled_topk_kernel<TopK>;
constexpr size_t smem_size =
vllm::filtered_topk::FILTERED_TOPK_SMEM_DYNAMIC;
cudaError_t status = cudaFuncSetAttribute(
kernel, cudaFuncAttributeMaxDynamicSharedMemorySize, smem_size);
STD_TORCH_CHECK(status == cudaSuccess,
"sampled_topk smem failed: ", cudaGetErrorString(status));
kernel<<<num_rows, vllm::sampled_topk::kThreads, smem_size, stream>>>(
logits.const_data_ptr<float>(), lengths.const_data_ptr<int32_t>(),
output.mutable_data_ptr<int32_t>(), stride,
static_cast<int>(std::min(max_seq_len, logits.size(1))));
} else if (num_rows > 32 && max_smem_per_block >= 128 * 1024) {
cudaError_t status =
vllm::FilteredTopKRaggedTransform<float, int32_t, TopK>(
logits.const_data_ptr<float>(), output.mutable_data_ptr<int32_t>(),
+100
View File
@@ -932,6 +932,106 @@ def test_deepseek_workspace_topk(
)
@pytest.mark.skipif(not _has_device_capability(80), reason="Requires SM80+")
@pytest.mark.parametrize("rows", [65, 128])
@pytest.mark.parametrize("top_k", [512, 1024, 2048])
@pytest.mark.parametrize(
"distribution", ["random", "10LSBits", "ascending", "constant", "sampled_peaks"]
)
@torch.inference_mode()
def test_persistent_topk_sampled_graph(
rows: int, top_k: int, distribution: str
) -> None:
"""Sampling and both fallbacks preserve exact values across graph replays."""
if torch.cuda.get_device_properties(0).shared_memory_per_block_optin < 144 * 1024:
pytest.skip("Sampled top-k requires at least 144 KiB of shared memory")
set_random_seed(42)
min_sampled_length = 98304 if top_k == 512 else 65536
width = min_sampled_length + 3
lengths = torch.full((rows,), width, dtype=torch.int32, device="cuda")
logits = create_random_logits(
torch.zeros_like(lengths),
lengths,
torch.float32,
42,
False,
"10LSBits" if distribution == "10LSBits" else "random",
)
if distribution == "ascending":
logits.copy_(torch.arange(width, device="cuda", dtype=torch.float32))
elif distribution == "constant":
logits.fill_(1.0)
elif distribution == "sampled_peaks":
# A biased sample leaves fewer than k survivors and must fall back.
logits.zero_()
sample = torch.arange(4096, device="cuda")
positions = (sample // 32) * (width // 128) + sample % 32
logits[:, positions] = 1 + sample.float() / 4096
original = logits.clone()
indices = torch.empty((rows, top_k), dtype=torch.int32, device="cuda")
workspace = torch.empty(RADIX_TOPK_WORKSPACE_SIZE, dtype=torch.uint8, device="cuda")
def run() -> None:
torch.ops._C.persistent_topk(
logits,
lengths.view(-1, 4 if rows == 128 else 1),
indices,
workspace,
top_k,
width,
)
run()
graph = torch.cuda.CUDAGraph()
with torch.cuda.graph(graph):
run()
positions = torch.arange(width, device="cuda")
slots = torch.arange(top_k, device="cuda")
for step in range(3):
bounds = torch.tensor(
[
-1,
0,
1,
top_k - 1,
top_k,
32769,
min_sampled_length - 1,
min_sampled_length,
width,
width + 17,
],
dtype=torch.int32,
device="cuda",
)
lengths.copy_(
bounds[(torch.arange(rows, device="cuda") + step) % bounds.numel()]
)
logits.copy_(original if step != 1 else original.flip(1))
logits.masked_fill_(positions[None] >= lengths[:, None], float("nan"))
indices.fill_(-2)
graph.replay()
valid = slots[None] < lengths.clamp(max=top_k)[:, None]
assert torch.all(indices[~valid] == -1)
assert torch.all(((indices >= 0) & (indices < lengths[:, None])) == valid)
ordered_indices = indices.sort(dim=1).values
assert torch.all(
(ordered_indices[:, 1:] != ordered_indices[:, :-1])
| (ordered_indices[:, 1:] == -1)
)
selected = logits.gather(1, indices.clamp_min(0).long())
selected.masked_fill_(~valid, -float("inf"))
expected = logits.masked_fill(
positions[None] >= lengths[:, None], -float("inf")
)
torch.testing.assert_close(
selected.sort(dim=1, descending=True).values,
expected.topk(top_k, dim=1).values,
atol=0,
rtol=0,
)
def run_large_context_topk_test(
batch_size: int,
seq_lens: list[int],