[DSv4 Perf] DSv4 flashinfer sparse index cache for metadata, 2%~4% TTFT improvement (#45863)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -288,24 +288,40 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
query_start_loc = swa_metadata.query_start_loc[: num_reqs + 1]
|
||||
seq_lens = swa_metadata.seq_lens[:num_reqs]
|
||||
assert seq_lens.dtype == torch.int32
|
||||
sparse_indices, sparse_topk_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices,
|
||||
decode_compressed_indices,
|
||||
decode_compressed_topk_lens,
|
||||
prefill_topk_indices[:num_prefill_tokens],
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
swa_metadata.token_to_req_indices[:num_tokens],
|
||||
swa_metadata.block_table[:num_reqs],
|
||||
swa_metadata.block_size,
|
||||
compressed_block_table,
|
||||
compressed_block_size,
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
decode_compressed_indices_are_local=decode_compressed_indices_are_local,
|
||||
decode_is_valid_token=decode_is_valid_token,
|
||||
# cache for SWA-only and C128A that build the same mixed sparse indices
|
||||
# C4A stays uncached.
|
||||
cache_key = (
|
||||
"swa_only"
|
||||
if swa_only
|
||||
else ("c128a" if self.compress_ratio == 128 else "c4a")
|
||||
)
|
||||
cached_sparse = swa_metadata.flashinfer_sparse_index_cache.get(cache_key, None)
|
||||
if cached_sparse is None:
|
||||
sparse_indices, sparse_topk_lens = build_flashinfer_mixed_sparse_indices(
|
||||
decode_swa_indices,
|
||||
decode_compressed_indices,
|
||||
decode_compressed_topk_lens,
|
||||
prefill_topk_indices[:num_prefill_tokens],
|
||||
query_start_loc,
|
||||
seq_lens,
|
||||
swa_metadata.token_to_req_indices[:num_tokens],
|
||||
swa_metadata.block_table[:num_reqs],
|
||||
swa_metadata.block_size,
|
||||
compressed_block_table,
|
||||
compressed_block_size,
|
||||
self.window_size,
|
||||
self.compress_ratio,
|
||||
top_k,
|
||||
decode_compressed_indices_are_local=decode_compressed_indices_are_local,
|
||||
decode_is_valid_token=decode_is_valid_token,
|
||||
)
|
||||
if cache_key != "c4a":
|
||||
swa_metadata.flashinfer_sparse_index_cache[cache_key] = (
|
||||
sparse_indices,
|
||||
sparse_topk_lens,
|
||||
)
|
||||
else:
|
||||
sparse_indices, sparse_topk_lens = cached_sparse
|
||||
return compressed_kv_cache, seq_lens, sparse_indices, sparse_topk_lens
|
||||
|
||||
def _forward(
|
||||
|
||||
Reference in New Issue
Block a user