[HiSparse] Read GPU-resident KV directly for mixed and BF16 prefills (#60417)

Signed-off-by: Matthew Bonanni <mbonanni@redhat.com>
Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Matthew Bonanni
2026-10-13 16:40:20 -04:00
committed by GitHub
co-authored by Claude Opus 5.5
parent c19c263ac8
commit 292da21f9d
5 changed files with 165 additions and 86 deletions
@@ -122,6 +122,26 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
index_group = self.index_group
if isinstance(index_group, HiSparseMLAIndexGroup):
num_decode_tokens = attn_metadata.num_decode_tokens
physical_kv_cache = index_group.physical_kv_cache(self.index_group_index)
if (
num_decode_tokens == num_actual_toks
or index_group.cache(self.index_group_index).all_context_pages_resident
):
topk_indices_physical = cast(
torch.Tensor,
index_group.convert_logical_to_physical_topk(
self.index_group_index,
topk_indices,
attn_metadata,
block_stride_rows=None,
return_valid_counts=False,
),
)
return (
self._run_mqa_kernel(q, physical_kv_cache, topk_indices_physical),
None,
)
outputs = []
if num_decode_tokens:
topk_indices_physical = cast(
@@ -137,51 +157,32 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
outputs.append(
self._run_mqa_kernel(
q[:num_decode_tokens],
index_group.physical_kv_cache(self.index_group_index),
physical_kv_cache,
topk_indices_physical,
)
)
if num_decode_tokens < num_actual_toks:
cache = index_group.cache(self.index_group_index)
if num_decode_tokens == 0 and cache.all_context_pages_resident:
topk_indices_physical = cast(
torch.Tensor,
index_group.convert_logical_to_physical_topk(
self.index_group_index,
topk_indices,
attn_metadata,
block_stride_rows=None,
return_valid_counts=False,
),
)
prefill_cache = index_group.physical_kv_cache(
self.index_group_index
)
else:
prefill_cache, block_table, req_ids = (
index_group.stage_prefill_rows(
self.index_group_index,
kv_c_and_k_pe_cache,
attn_metadata,
)
)
topk_indices_physical = cast(
torch.Tensor,
triton_convert_req_index_to_global_index(
req_ids,
block_table,
topk_indices[num_decode_tokens:],
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
),
)
outputs.append(
self._run_mqa_kernel(
q[num_decode_tokens:],
prefill_cache,
topk_indices_physical,
)
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
self.index_group_index,
kv_c_and_k_pe_cache,
attn_metadata,
)
topk_indices_physical = cast(
torch.Tensor,
triton_convert_req_index_to_global_index(
req_ids,
block_table,
topk_indices[num_decode_tokens:],
BLOCK_SIZE=attn_metadata.block_size,
NUM_TOPK_TOKENS=topk_indices.shape[1],
),
)
outputs.append(
self._run_mqa_kernel(
q[num_decode_tokens:],
prefill_cache,
topk_indices_physical,
)
)
output = torch.cat(outputs) if len(outputs) > 1 else outputs[0]
return output, None
@@ -852,6 +852,21 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
if cache is not None:
assert isinstance(index_group, HiSparseMLAIndexGroup)
num_decode_tokens = attn_metadata.num_decode_tokens
if num_decode_tokens < q.shape[0] and cache.all_context_pages_resident:
physical_topk, lengths = index_group.convert_logical_to_physical_topk(
self.index_group_index,
topk_indices,
attn_metadata,
block_stride_rows=None,
return_valid_counts=True,
)
return self._bf16_flash_mla_kernel(
q,
index_group.physical_kv_cache(self.index_group_index),
physical_topk,
lengths,
actual_num_heads,
)
if num_decode_tokens > 0:
decode_topk, decode_lengths = (
index_group.convert_logical_to_physical_topk(
@@ -233,7 +233,10 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
cache = self.cache(layer_index)
num_tokens = logical_topk_indices.shape[0]
req_id_per_token = attn_metadata.req_id_per_token[:num_tokens]
if num_tokens > cache.runtime.max_swap_rows:
if num_tokens > cache.runtime.max_swap_rows or (
cache.all_context_pages_resident
and cache.num_decode_tokens < cache.num_actual_tokens
):
# Prefill-sized batches do not fit the decode residency workspace.
# Non-resident prefills are staged before reaching this path.
assert cache.all_context_pages_resident