[HiSparse] Own GPU residency instead of editing core block-table rows (#60647)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-09 21:25:44 -07:00
committed by GitHub
co-authored by Claude Opus 5.5
parent 9bd074a996
commit d5f356dae0
15 changed files with 429 additions and 128 deletions
+12 -12
View File
@@ -194,7 +194,7 @@ constexpr int kHashLoadInverse = 4;
// the page is not resident) and returns its host row (-1 when not host-backed).
__device__ __forceinline__ int32_t translate_topk_entry(
const int32_t token_index, const int32_t request_row,
const int32_t* __restrict__ source_block_table,
const int32_t state_row, const int32_t* __restrict__ source_block_table,
const int32_t* __restrict__ resident_block_table, const int64_t host_rows,
const int64_t source_bt_stride, const int32_t source_num_reqs,
const int32_t source_num_blocks, const int32_t source_block_size,
@@ -221,10 +221,10 @@ __device__ __forceinline__ int32_t translate_topk_entry(
if (resident_block_table != nullptr) {
const int32_t resident_block =
token_index >= 0 ? token_index / resident_block_size : -1;
if (request_row >= 0 && request_row < resident_num_reqs &&
if (state_row >= 0 && state_row < resident_num_reqs &&
resident_block >= 0 && resident_block < resident_num_blocks) {
const int32_t physical_block =
resident_block_table[static_cast<int64_t>(request_row) *
resident_block_table[static_cast<int64_t>(state_row) *
resident_bt_stride +
resident_block];
if (physical_block != resident_null_block && physical_block >= 0) {
@@ -424,11 +424,11 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
for (int i = tid; i < top_k; i += blockDim.x) {
int32_t resident_row;
const int32_t g = translate_topk_entry(
row_topk[i], request_row, source_block_table, resident_block_table,
host_rows, source_bt_stride, source_num_reqs, source_num_blocks,
source_block_size, resident_bt_stride, resident_num_reqs,
resident_num_blocks, resident_block_size, resident_null_block,
resident_row);
row_topk[i], request_row, state_row, source_block_table,
resident_block_table, host_rows, source_bt_stride, source_num_reqs,
source_num_blocks, source_block_size, resident_bt_stride,
resident_num_reqs, resident_num_blocks, resident_block_size,
resident_null_block, resident_row);
if (resolved_global_indices != nullptr) {
resolved_global_indices[static_cast<int64_t>(row) * top_k + i] = g;
}
@@ -602,8 +602,8 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
g = translate_topk_entry(
global_indices[static_cast<int64_t>(row) * input_row_stride +
position % top_k],
request_row, source_block_table, resident_block_table, host_rows,
source_bt_stride, source_num_reqs, source_num_blocks,
request_row, state_row, source_block_table, resident_block_table,
host_rows, source_bt_stride, source_num_reqs, source_num_blocks,
source_block_size, resident_bt_stride, resident_num_reqs,
resident_num_blocks, resident_block_size, resident_null_block,
resident_row);
@@ -708,8 +708,8 @@ __global__ __launch_bounds__(1024) void hisparse_resolve_residency_kernel(
const int32_t g = translate_topk_entry(
global_indices[static_cast<int64_t>(row) * input_row_stride +
position % top_k],
request_row, source_block_table, resident_block_table, host_rows,
source_bt_stride, source_num_reqs, source_num_blocks,
request_row, state_row, source_block_table, resident_block_table,
host_rows, source_bt_stride, source_num_reqs, source_num_blocks,
source_block_size, resident_bt_stride, resident_num_reqs,
resident_num_blocks, resident_block_size, resident_null_block,
resident_row);