[3/N] HiSparse: host-resident sparse-MLA decode hot-buffering (#53781)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
Signed-off-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Signed-off-by: Matthew Bonanni <matthew@neuralmagic.com>
Signed-off-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <codex@openai.com>
Co-authored-by: Lucas Wilkinson <lwilkins@redhat.com>
Co-authored-by: Lucas Wilkinson <lwilkinson@neuralmagic.com>
Co-authored-by: Nicolò Lucchesi <nicolo.lucchesi@mistral.ai>
Co-authored-by: OpenAI Codex <noreply@openai.com>
Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
Co-authored-by: Lucas Wilkinson <LucasWilkinson@users.noreply.github.com>
This commit is contained in:
Matthew Bonanni
2026-09-12 00:36:47 -04:00
committed by GitHub
co-authored by OpenAI Codex Lucas Wilkinson Lucas Wilkinson Nicolò Lucchesi OpenAI Codex Claude Opus 5 Lucas Wilkinson
parent 120ec4ebd2
commit d43bb2f37f
124 changed files with 14262 additions and 906 deletions
+70 -5
View File
@@ -1458,7 +1458,11 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache_page(
const int32_t block_size, const int32_t total_tokens,
const int64_t block_table_stride, const int64_t cache_block_stride,
const int64_t cache_entry_stride, const int64_t dst_entry_stride,
const int32_t* __restrict__ seq_starts) {
const int32_t* __restrict__ seq_starts,
const uint8_t* __restrict__ host_cache,
const int32_t* __restrict__ host_row_ids,
const int32_t* __restrict__ device_row_ids,
const int64_t host_entry_stride) {
constexpr int32_t warps_per_cta = 16;
__shared__ GatherPageTask page;
__shared__ int32_t physical_block;
@@ -1484,8 +1488,22 @@ __global__ void cp_gather_and_upconvert_fp8_kv_cache_page(
page_token < page.page_token_end; page_token += warps_per_cta) {
const int32_t output_token =
page.output_token_begin + page_token - page.page_token_begin;
const uint8_t* token_ptr = src_cache + physical_block * cache_block_stride +
page_token * cache_entry_stride;
const int32_t source_row = physical_block * block_size + page_token;
const uint8_t* token_ptr;
if (device_row_ids == nullptr) {
token_ptr = src_cache + physical_block * cache_block_stride +
page_token * cache_entry_stride;
} else {
const int32_t device_row = device_row_ids[source_row];
if (device_row >= 0) {
token_ptr = src_cache + (device_row / block_size) * cache_block_stride +
(device_row % block_size) * cache_entry_stride;
} else {
token_ptr =
host_cache +
static_cast<int64_t>(host_row_ids[source_row]) * host_entry_stride;
}
}
__nv_bfloat16* dst_ptr = dst + output_token * dst_entry_stride;
gather_and_upconvert_fp8_token(token_ptr, dst_ptr, lane_id);
}
@@ -1694,7 +1712,10 @@ void cp_gather_and_upconvert_fp8_kv_cache(
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
torch::stable::Tensor const& workspace_starts, // [BATCH]
int64_t batch_size,
std::optional<torch::stable::Tensor> seq_starts = std::nullopt) {
std::optional<torch::stable::Tensor> seq_starts = std::nullopt,
std::optional<torch::stable::Tensor> host_cache = std::nullopt,
std::optional<torch::stable::Tensor> host_row_ids = std::nullopt,
std::optional<torch::stable::Tensor> device_row_ids = std::nullopt) {
torch::stable::accelerator::DeviceGuard device_guard(
src_cache.get_device_index());
const cudaStream_t stream = get_current_cuda_stream();
@@ -1714,6 +1735,12 @@ void cp_gather_and_upconvert_fp8_kv_cache(
"seq_starts must be int32");
}
const bool has_host_rows = host_cache.has_value();
STD_TORCH_CHECK(
has_host_rows == host_row_ids.has_value() &&
has_host_rows == device_row_ids.has_value(),
"host_cache, host_row_ids, and device_row_ids must be provided together");
STD_TORCH_CHECK(src_cache.device() == dst.device(),
"src_cache and dst must be on the same device");
STD_TORCH_CHECK(src_cache.device() == block_table.device(),
@@ -1724,6 +1751,43 @@ void cp_gather_and_upconvert_fp8_kv_cache(
STD_TORCH_CHECK(src_cache.device() == seq_starts.value().device(),
"src_cache and seq_starts must be on the same device");
}
const uint8_t* host_cache_ptr = nullptr;
const int32_t* host_row_ids_ptr = nullptr;
const int32_t* device_row_ids_ptr = nullptr;
int64_t host_entry_stride = 0;
if (has_host_rows) {
auto const& host = host_cache.value();
auto const& host_rows = host_row_ids.value();
auto const& device_rows = device_row_ids.value();
STD_TORCH_CHECK(
host.device().is_cpu() &&
host.scalar_type() == torch::headeronly::ScalarType::Byte &&
host.dim() == 2 && host.is_contiguous() &&
host.size(1) == src_cache.size(2),
"host_cache must be a contiguous uint8 CPU row matrix matching "
"src_cache");
cudaPointerAttributes attributes{};
const cudaError_t pointer_status =
cudaPointerGetAttributes(&attributes, host.const_data_ptr());
if (pointer_status != cudaSuccess) {
cudaGetLastError();
}
STD_TORCH_CHECK(
pointer_status == cudaSuccess && attributes.type == cudaMemoryTypeHost,
"host_cache must be CUDA-accessible pinned memory");
STD_TORCH_CHECK(
host_rows.is_cuda() && device_rows.is_cuda() &&
host_rows.scalar_type() == torch::headeronly::ScalarType::Int &&
device_rows.scalar_type() == torch::headeronly::ScalarType::Int &&
host_rows.is_contiguous() && device_rows.is_contiguous() &&
host_rows.numel() == device_rows.numel() && host_rows.dim() <= 2 &&
device_rows.dim() <= 2,
"host/device row maps must be matching contiguous CUDA int32 tensors");
host_cache_ptr = reinterpret_cast<const uint8_t*>(host.const_data_ptr());
host_row_ids_ptr = host_rows.const_data_ptr<int32_t>();
device_row_ids_ptr = device_rows.const_data_ptr<int32_t>();
host_entry_stride = host.stride(0);
}
auto dtype = src_cache.scalar_type();
STD_TORCH_CHECK(
dtype == torch::headeronly::ScalarType::Byte || // uint8
@@ -1766,7 +1830,8 @@ void cp_gather_and_upconvert_fp8_kv_cache(
workspace_starts.const_data_ptr<int32_t>(),
static_cast<int32_t>(batch_size), block_size, total_tokens,
block_table_stride, cache_block_stride, cache_entry_stride,
dst_entry_stride, seq_starts_ptr);
dst_entry_stride, seq_starts_ptr, host_cache_ptr, host_row_ids_ptr,
device_row_ids_ptr, host_entry_stride);
}
void cp_gather_and_upconvert_nvfp4_kv_cache(
File diff suppressed because it is too large Load Diff
+52 -1
View File
@@ -661,6 +661,54 @@ void concat_and_cache_mla_grouped(
std::optional<torch::stable::Tensor> kv_scales,
const std::string& kv_cache_dtype);
#ifndef USE_ROCM
// HiSparse kernels use raw PTX in the row copy; CUDA-only.
void hisparse_resolve_residency(
torch::stable::Tensor const& host_cache, torch::stable::Tensor& hot_cache,
torch::stable::Tensor const& hot_block_table,
torch::stable::Tensor const& global_indices,
torch::stable::Tensor& hot_indices,
torch::stable::Tensor& device_global_indices,
torch::stable::Tensor& lru_slots,
std::optional<torch::stable::Tensor> const& request_state_indices,
int64_t region_stride,
std::optional<torch::stable::Tensor> const& miss_mask,
std::optional<torch::stable::Tensor> const& attention_indices,
int64_t attention_block_stride,
std::optional<torch::stable::Tensor> const& request_ids,
std::optional<torch::stable::Tensor> const& source_block_table,
int64_t source_block_size,
std::optional<torch::stable::Tensor> const& resolved_global_indices,
std::optional<torch::stable::Tensor> const& valid_counts,
std::optional<torch::stable::Tensor> const& swap_host_physical_rows,
std::optional<torch::stable::Tensor> const& swap_device_physical_rows,
std::optional<torch::stable::Tensor> const& swap_counts,
std::optional<torch::stable::Tensor> const& resident_block_table,
int64_t resident_block_size, int64_t resident_null_block);
void hisparse_invalidate_written_slots(
torch::stable::Tensor& device_global_indices,
torch::stable::Tensor const& request_state_indices,
torch::stable::Tensor const& req_id_per_token,
torch::stable::Tensor const& written_slots);
void hisparse_gather_plan(
torch::stable::Tensor const& host_cache, torch::stable::Tensor& hot_cache,
torch::stable::Tensor const& global_indices,
torch::stable::Tensor const& hot_indices,
torch::stable::Tensor const& miss_mask,
std::optional<torch::stable::Tensor> const& request_state_indices,
std::optional<torch::stable::Tensor> const& attention_indices,
int64_t attention_block_stride);
void hisparse_gather_compact(torch::stable::Tensor const& host_cache,
torch::stable::Tensor& hot_cache,
torch::stable::Tensor const& miss_global_indices,
torch::stable::Tensor const& miss_hot_indices,
torch::stable::Tensor const& miss_counts);
#endif // !USE_ROCM
// NOTE: k_pe and kv_c order is flipped compared to concat_and_cache_mla
void concat_and_cache_mla_rope_fused(
torch::stable::Tensor& positions, torch::stable::Tensor& q_pe,
@@ -704,7 +752,10 @@ void cp_gather_and_upconvert_fp8_kv_cache(
torch::stable::Tensor const& block_table, // [BATCH, BLOCK_INDICES]
torch::stable::Tensor const& workspace_starts, // [BATCH]
int64_t batch_size,
std::optional<torch::stable::Tensor> seq_starts = std::nullopt);
std::optional<torch::stable::Tensor> seq_starts = std::nullopt,
std::optional<torch::stable::Tensor> host_cache = std::nullopt,
std::optional<torch::stable::Tensor> host_row_ids = std::nullopt,
std::optional<torch::stable::Tensor> device_row_ids = std::nullopt);
// Gather and upconvert an nvfp4_ds_mla KV cache to a BF16 workspace
void cp_gather_and_upconvert_nvfp4_kv_cache(
+62 -1
View File
@@ -987,6 +987,57 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
" Tensor? kv_scales=None,"
" str kv_cache_dtype='auto') -> ()");
#ifndef USE_ROCM
ops.def(
"hisparse_resolve_residency(Tensor host_cache,"
" Tensor! hot_cache,"
" Tensor hot_block_table,"
" Tensor global_indices,"
" Tensor! hot_indices,"
" Tensor! device_global_indices,"
" Tensor! lru_slots,"
" Tensor? request_state_indices,"
" int region_stride,"
" Tensor(a!)? miss_mask=None,"
" Tensor(b!)? attention_indices=None,"
" int attention_block_stride=0,"
" Tensor? request_ids=None,"
" Tensor? source_block_table=None,"
" int source_block_size=0,"
" Tensor(c!)? resolved_global_indices=None,"
" Tensor(d!)? valid_counts=None,"
" Tensor(e!)? swap_host_physical_rows=None,"
" Tensor(f!)? swap_device_physical_rows=None,"
" Tensor(g!)? swap_counts=None,"
" Tensor? resident_block_table=None,"
" int resident_block_size=0,"
" int resident_null_block=0) -> ()");
ops.def(
"hisparse_invalidate_written_slots(Tensor! device_global_indices,"
" Tensor request_state_indices,"
" Tensor req_id_per_token,"
" Tensor written_slots) -> ()");
ops.def(
"hisparse_gather_plan(Tensor host_cache,"
" Tensor! hot_cache,"
" Tensor global_indices,"
" Tensor hot_indices,"
" Tensor miss_mask,"
" Tensor? request_state_indices,"
" Tensor(a!)? attention_indices=None,"
" int attention_block_stride=0) -> ()");
ops.def(
"hisparse_gather_compact(Tensor host_cache,"
" Tensor! hot_cache,"
" Tensor miss_global_indices,"
" Tensor miss_hot_indices,"
" Tensor miss_counts) -> ()");
#endif // !USE_ROCM
// Rotate Q and K, then write to kv cache for MLA
ops.def(
"concat_and_cache_mla_rope_fused("
@@ -1023,7 +1074,8 @@ STABLE_TORCH_LIBRARY_FRAGMENT(_C_cache_ops, ops) {
ops.def(
"cp_gather_and_upconvert_fp8_kv_cache(Tensor src_cache, Tensor! dst, "
"Tensor block_table, Tensor workspace_starts, int batch_size, Tensor? "
"seq_starts) -> ()");
"seq_starts, Tensor? host_cache=None, Tensor? host_row_ids=None, Tensor? "
"device_row_ids=None) -> ()");
ops.def(
"cp_gather_and_upconvert_nvfp4_kv_cache(Tensor src_cache, Tensor! dst, "
@@ -1091,6 +1143,15 @@ STABLE_TORCH_LIBRARY_IMPL(_C_cache_ops, CUDA, ops) {
ops.impl("concat_and_cache_mla", TORCH_BOX(&concat_and_cache_mla));
ops.impl("concat_and_cache_mla_grouped",
TORCH_BOX(&concat_and_cache_mla_grouped));
#ifndef USE_ROCM
ops.impl("hisparse_resolve_residency",
TORCH_BOX(&hisparse_resolve_residency));
ops.impl("hisparse_invalidate_written_slots",
TORCH_BOX(&hisparse_invalidate_written_slots));
ops.impl("hisparse_gather_plan", TORCH_BOX(&hisparse_gather_plan));
ops.impl("hisparse_gather_compact", TORCH_BOX(&hisparse_gather_compact));
#endif // !USE_ROCM
ops.impl("concat_and_cache_mla_rope_fused",
TORCH_BOX(&concat_and_cache_mla_rope_fused));
ops.impl("convert_fp8", TORCH_BOX(&convert_fp8));