[Kernel][Backend][Model] Blocksparse flash attention kernel and Phi-3-Small model (#4799)
Co-authored-by: beagleski <yunanzhang@microsoft.com> Co-authored-by: bapatra <bapatra@microsoft.com> Co-authored-by: Barun Patra <codedecde@users.noreply.github.com> Co-authored-by: Michael Goin <michael@neuralmagic.com>
This commit is contained in:
co-authored by
beagleski
bapatra
Barun Patra
Michael Goin
parent
e64fde4b01
commit
8e192ff967
+21
-9
@@ -45,11 +45,17 @@ def paged_attention_v1(
|
||||
alibi_slopes: Optional[torch.Tensor],
|
||||
kv_cache_dtype: str,
|
||||
kv_scale: float,
|
||||
tp_rank: int = 0,
|
||||
blocksparse_local_blocks: int = 0,
|
||||
blocksparse_vert_stride: int = 0,
|
||||
blocksparse_block_size: int = 64,
|
||||
blocksparse_head_sliding_step: int = 0,
|
||||
) -> None:
|
||||
vllm_ops.paged_attention_v1(out, query, key_cache, value_cache,
|
||||
num_kv_heads, scale, block_tables, seq_lens,
|
||||
block_size, max_seq_len, alibi_slopes,
|
||||
kv_cache_dtype, kv_scale)
|
||||
vllm_ops.paged_attention_v1(
|
||||
out, query, key_cache, value_cache, num_kv_heads, scale, block_tables,
|
||||
seq_lens, block_size, max_seq_len, alibi_slopes, kv_cache_dtype,
|
||||
kv_scale, tp_rank, blocksparse_local_blocks, blocksparse_vert_stride,
|
||||
blocksparse_block_size, blocksparse_head_sliding_step)
|
||||
|
||||
|
||||
def paged_attention_v2(
|
||||
@@ -69,12 +75,18 @@ def paged_attention_v2(
|
||||
alibi_slopes: Optional[torch.Tensor],
|
||||
kv_cache_dtype: str,
|
||||
kv_scale: float,
|
||||
tp_rank: int = 0,
|
||||
blocksparse_local_blocks: int = 0,
|
||||
blocksparse_vert_stride: int = 0,
|
||||
blocksparse_block_size: int = 64,
|
||||
blocksparse_head_sliding_step: int = 0,
|
||||
) -> None:
|
||||
vllm_ops.paged_attention_v2(out, exp_sum, max_logits, tmp_out, query,
|
||||
key_cache, value_cache, num_kv_heads, scale,
|
||||
block_tables, seq_lens, block_size,
|
||||
max_seq_len, alibi_slopes, kv_cache_dtype,
|
||||
kv_scale)
|
||||
vllm_ops.paged_attention_v2(
|
||||
out, exp_sum, max_logits, tmp_out, query, key_cache, value_cache,
|
||||
num_kv_heads, scale, block_tables, seq_lens, block_size, max_seq_len,
|
||||
alibi_slopes, kv_cache_dtype, kv_scale, tp_rank,
|
||||
blocksparse_local_blocks, blocksparse_vert_stride,
|
||||
blocksparse_block_size, blocksparse_head_sliding_step)
|
||||
|
||||
|
||||
# pos encoding ops
|
||||
|
||||
Reference in New Issue
Block a user