[HiSparse] Read GPU-resident KV directly for mixed and BF16 prefills (#60417)
Signed-off-by: Matthew Bonanni <mbonanni@redhat.com> Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5.5
parent
c19c263ac8
commit
292da21f9d
@@ -802,6 +802,27 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
kv_c_and_k_pe_cache.dtype
|
||||
)
|
||||
assert hot_cache.is_contiguous(), "HiSparse hot buffer must be contiguous"
|
||||
if (
|
||||
num_decode_tokens >= num_tokens
|
||||
or index_group.cache(layer_index).all_context_pages_resident
|
||||
):
|
||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||
layer_index,
|
||||
topk,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
return self._forward_mqa_kernel(
|
||||
q,
|
||||
hot_cache,
|
||||
physical_topk,
|
||||
valid_counts,
|
||||
layer=layer,
|
||||
block_size=block_size,
|
||||
is_decode=num_decode_tokens >= num_tokens,
|
||||
)
|
||||
|
||||
decode_out = decode_lse = None
|
||||
if num_decode_tokens:
|
||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||
@@ -825,31 +846,18 @@ class SparseMLACommonImpl(MLACommonBaseImpl[T], SharedTopkIndicesBuffer, Generic
|
||||
block_size=block_size,
|
||||
is_decode=True,
|
||||
)
|
||||
if num_decode_tokens >= num_tokens:
|
||||
return decode_out, decode_lse
|
||||
|
||||
cache = index_group.cache(layer_index)
|
||||
if num_decode_tokens == 0 and cache.all_context_pages_resident:
|
||||
physical_topk, valid_counts = index_group.convert_logical_to_physical_topk(
|
||||
layer_index,
|
||||
topk,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
prefill_cache = hot_cache
|
||||
else:
|
||||
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
||||
layer_index, kv_c_and_k_pe_cache, attn_metadata
|
||||
)
|
||||
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
topk[num_decode_tokens:],
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
||||
layer_index, kv_c_and_k_pe_cache, attn_metadata
|
||||
)
|
||||
physical_topk, valid_counts = triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
topk[num_decode_tokens:],
|
||||
BLOCK_SIZE=block_size,
|
||||
NUM_TOPK_TOKENS=topk.shape[1],
|
||||
return_valid_counts=True,
|
||||
)
|
||||
q_prefill = (
|
||||
(q[0][num_decode_tokens:], q[1][num_decode_tokens:])
|
||||
if isinstance(q, tuple)
|
||||
|
||||
@@ -122,6 +122,26 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
|
||||
index_group = self.index_group
|
||||
if isinstance(index_group, HiSparseMLAIndexGroup):
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
physical_kv_cache = index_group.physical_kv_cache(self.index_group_index)
|
||||
if (
|
||||
num_decode_tokens == num_actual_toks
|
||||
or index_group.cache(self.index_group_index).all_context_pages_resident
|
||||
):
|
||||
topk_indices_physical = cast(
|
||||
torch.Tensor,
|
||||
index_group.convert_logical_to_physical_topk(
|
||||
self.index_group_index,
|
||||
topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=False,
|
||||
),
|
||||
)
|
||||
return (
|
||||
self._run_mqa_kernel(q, physical_kv_cache, topk_indices_physical),
|
||||
None,
|
||||
)
|
||||
|
||||
outputs = []
|
||||
if num_decode_tokens:
|
||||
topk_indices_physical = cast(
|
||||
@@ -137,51 +157,32 @@ class FlashInferMLASparseSM120Impl(SparseMLACommonImpl[FlashInferMLASparseMetada
|
||||
outputs.append(
|
||||
self._run_mqa_kernel(
|
||||
q[:num_decode_tokens],
|
||||
index_group.physical_kv_cache(self.index_group_index),
|
||||
physical_kv_cache,
|
||||
topk_indices_physical,
|
||||
)
|
||||
)
|
||||
if num_decode_tokens < num_actual_toks:
|
||||
cache = index_group.cache(self.index_group_index)
|
||||
if num_decode_tokens == 0 and cache.all_context_pages_resident:
|
||||
topk_indices_physical = cast(
|
||||
torch.Tensor,
|
||||
index_group.convert_logical_to_physical_topk(
|
||||
self.index_group_index,
|
||||
topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=False,
|
||||
),
|
||||
)
|
||||
prefill_cache = index_group.physical_kv_cache(
|
||||
self.index_group_index
|
||||
)
|
||||
else:
|
||||
prefill_cache, block_table, req_ids = (
|
||||
index_group.stage_prefill_rows(
|
||||
self.index_group_index,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
)
|
||||
)
|
||||
topk_indices_physical = cast(
|
||||
torch.Tensor,
|
||||
triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
topk_indices[num_decode_tokens:],
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
),
|
||||
)
|
||||
outputs.append(
|
||||
self._run_mqa_kernel(
|
||||
q[num_decode_tokens:],
|
||||
prefill_cache,
|
||||
topk_indices_physical,
|
||||
)
|
||||
prefill_cache, block_table, req_ids = index_group.stage_prefill_rows(
|
||||
self.index_group_index,
|
||||
kv_c_and_k_pe_cache,
|
||||
attn_metadata,
|
||||
)
|
||||
topk_indices_physical = cast(
|
||||
torch.Tensor,
|
||||
triton_convert_req_index_to_global_index(
|
||||
req_ids,
|
||||
block_table,
|
||||
topk_indices[num_decode_tokens:],
|
||||
BLOCK_SIZE=attn_metadata.block_size,
|
||||
NUM_TOPK_TOKENS=topk_indices.shape[1],
|
||||
),
|
||||
)
|
||||
outputs.append(
|
||||
self._run_mqa_kernel(
|
||||
q[num_decode_tokens:],
|
||||
prefill_cache,
|
||||
topk_indices_physical,
|
||||
)
|
||||
)
|
||||
output = torch.cat(outputs) if len(outputs) > 1 else outputs[0]
|
||||
return output, None
|
||||
|
||||
|
||||
@@ -852,6 +852,21 @@ class FlashMLASparseImpl(SparseMLACommonImpl[FlashMLASparseMetadata]):
|
||||
if cache is not None:
|
||||
assert isinstance(index_group, HiSparseMLAIndexGroup)
|
||||
num_decode_tokens = attn_metadata.num_decode_tokens
|
||||
if num_decode_tokens < q.shape[0] and cache.all_context_pages_resident:
|
||||
physical_topk, lengths = index_group.convert_logical_to_physical_topk(
|
||||
self.index_group_index,
|
||||
topk_indices,
|
||||
attn_metadata,
|
||||
block_stride_rows=None,
|
||||
return_valid_counts=True,
|
||||
)
|
||||
return self._bf16_flash_mla_kernel(
|
||||
q,
|
||||
index_group.physical_kv_cache(self.index_group_index),
|
||||
physical_topk,
|
||||
lengths,
|
||||
actual_num_heads,
|
||||
)
|
||||
if num_decode_tokens > 0:
|
||||
decode_topk, decode_lengths = (
|
||||
index_group.convert_logical_to_physical_topk(
|
||||
|
||||
@@ -233,7 +233,10 @@ class HiSparseMLAIndexGroup(SparseMLAIndexGroup):
|
||||
cache = self.cache(layer_index)
|
||||
num_tokens = logical_topk_indices.shape[0]
|
||||
req_id_per_token = attn_metadata.req_id_per_token[:num_tokens]
|
||||
if num_tokens > cache.runtime.max_swap_rows:
|
||||
if num_tokens > cache.runtime.max_swap_rows or (
|
||||
cache.all_context_pages_resident
|
||||
and cache.num_decode_tokens < cache.num_actual_tokens
|
||||
):
|
||||
# Prefill-sized batches do not fit the decode residency workspace.
|
||||
# Non-resident prefills are staged before reaching this path.
|
||||
assert cache.all_context_pages_resident
|
||||
|
||||
Reference in New Issue
Block a user