Use FlashAttention for multi_query_kv_attention (#4)

This commit is contained in:
Woosuk Kwon
2023-03-01 21:13:08 -08:00
committed by GitHub
parent 0deacbce6e
commit 3e9f991d6a
4 changed files with 107 additions and 33 deletions
+65 -2
View File
@@ -1,10 +1,13 @@
import random
from typing import Optional
from flash_attn.flash_attention import FlashAttention
import torch
from cacheflow import attention_ops
MAX_SEQ_LEN = 4096
def ref_masked_attention(
query: torch.Tensor,
@@ -79,7 +82,7 @@ def test_single_query_cached_kv_attention(
value_cache = torch.randn(
size=(num_blocks, *value_block_shape), dtype=dtype, device='cuda')
context_lens = [random.randint(1, 4096) for _ in range(num_tokens)]
context_lens = [random.randint(1, MAX_SEQ_LEN) for _ in range(num_tokens)]
max_context_len = max(context_lens)
context_lens = torch.tensor(context_lens, dtype=torch.int, device='cuda')
@@ -123,11 +126,60 @@ def test_single_query_cached_kv_attention(
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
def test_multi_query_kv_attention(
num_seqs: int,
num_heads: int,
head_size: int,
dtype: torch.dtype,
) -> None:
seq_lens = random.sample(range(1, MAX_SEQ_LEN), num_seqs)
max_seq_len = max(seq_lens)
num_tokens = sum(seq_lens)
cu_seq_lens = [0]
for seq_len in seq_lens:
cu_seq_lens.append(cu_seq_lens[-1] + seq_len)
cu_seq_lens = torch.tensor(cu_seq_lens, dtype=torch.int, device='cuda')
scale = float(1.0 / (head_size ** 0.5))
query = torch.randn(
num_tokens, num_heads, head_size, dtype=dtype, device='cuda')
key = torch.rand_like(query)
value = torch.rand_like(query)
qkv = torch.stack([query, key, value], dim=1)
flash_attn = FlashAttention(softmax_scale=scale)
output = flash_attn(
qkv,
cu_seqlens=cu_seq_lens,
max_s=max_seq_len,
causal=True,
)[0]
ref_outputs = []
for i, seq_len in enumerate(seq_lens):
attn_mask = torch.triu(torch.ones(seq_len, seq_len), diagonal=1) * -1e5
attn_mask = attn_mask.to(dtype=dtype, device='cuda')
start_idx = cu_seq_lens[i]
end_idx = cu_seq_lens[i + 1]
ref_output = ref_masked_attention(
query[start_idx:end_idx],
key[start_idx:end_idx],
value[start_idx:end_idx],
scale,
attn_mask=attn_mask,
)
ref_outputs.append(ref_output)
ref_output = torch.cat(ref_outputs, dim=0)
assert torch.allclose(output, ref_output, atol=1e-3, rtol=1e-5)
@torch.inference_mode()
def test_attention() -> None:
for dtype in [torch.half, torch.float]:
for block_size in [8, 16]:
for head_size in [64, 80, 96, 128, 256]:
for head_size in [32, 64, 80, 96, 128, 160, 192, 256]:
test_single_query_cached_kv_attention(
num_tokens=37,
num_heads=3,
@@ -137,6 +189,17 @@ def test_attention() -> None:
dtype=dtype,
)
# NOTE(woosuk): FlashAttention does not support FP32.
for dtype in [torch.half]:
# NOTE(woosuk): FlashAttention does not support head_size > 128.
for head_size in [64, 80, 96, 128]:
test_multi_query_kv_attention(
num_seqs=11,
num_heads=3,
head_size=head_size,
dtype=dtype,
)
if __name__ == '__main__':
test_attention()