[Dependency] Upgrade FlashInfer version to 0.7.0 (#58069)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com> Co-authored-by: Codex <noreply@openai.com> Co-authored-by: Yongye Zhu <zyy1102000@gmail.com> Co-authored-by: Kevin H. Luu <khluu000@gmail.com>
This commit is contained in:
co-authored by
Codex
Yongye Zhu
Kevin H. Luu
parent
a1b6763df6
commit
44bf011f44
@@ -1234,7 +1234,7 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
def _remap_flashinfer_index(values, block_size, block_span):
|
||||
# FlashInfer's DSv4 kernel indexes sparse KV by physical token stride, so
|
||||
# packed pages (#44577) need block*block_size+off -> block*block_span+off.
|
||||
# TODO: remove once flashinfer-ai/flashinfer#3856 is fixed.
|
||||
# The launcher declares these indices with sparse_indices_are_storage_offsets=True.
|
||||
is_valid = values >= 0
|
||||
safe_values = tl.where(is_valid, values, 0)
|
||||
values = (safe_values // block_size) * block_span
|
||||
|
||||
@@ -521,6 +521,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[:num_decode_tokens],
|
||||
sparse_indices_are_storage_offsets=True,
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=sparse_topk_lens[:num_decode_tokens],
|
||||
seq_lens=seq_lens[:num_decodes],
|
||||
@@ -546,6 +547,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[num_decode_tokens:num_tokens],
|
||||
sparse_indices_are_storage_offsets=True,
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=sparse_topk_lens[num_decode_tokens:num_tokens],
|
||||
seq_lens=seq_lens[num_decodes:num_reqs],
|
||||
|
||||
@@ -1266,7 +1266,7 @@ def build_flashinfer_mixed_sparse_indices(
|
||||
def _remap_flashinfer_index(values, block_size, block_span):
|
||||
# FlashInfer's DSv4 kernel indexes sparse KV by physical token stride, so
|
||||
# packed pages (#44577) need block*block_size+off -> block*block_span+off.
|
||||
# TODO: remove once flashinfer-ai/flashinfer#3856 is fixed.
|
||||
# The launcher declares these indices with sparse_indices_are_storage_offsets=True.
|
||||
is_valid = values >= 0
|
||||
safe_values = tl.where(is_valid, values, 0)
|
||||
values = (safe_values // block_size) * block_span
|
||||
|
||||
@@ -560,6 +560,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[:num_decode_tokens],
|
||||
sparse_indices_are_storage_offsets=True,
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=decode_topk_lens,
|
||||
seq_lens=decode_seq_lens,
|
||||
@@ -585,6 +586,7 @@ class DeepseekV4FlashInferMLAAttention(DeepseekV4Attention):
|
||||
swa_kv_cache=swa_k_cache,
|
||||
workspace_buffer=workspace,
|
||||
sparse_indices=sparse_indices[num_decode_tokens:num_tokens],
|
||||
sparse_indices_are_storage_offsets=True,
|
||||
compressed_kv_cache=compressed_kv_cache,
|
||||
sparse_topk_lens=sparse_topk_lens[num_decode_tokens:num_tokens],
|
||||
seq_lens=seq_lens[num_decodes:num_reqs],
|
||||
|
||||
Reference in New Issue
Block a user