Force paged attention v2 for long contexts (#1510)

This commit is contained in:
Antoni Baum
2023-11-01 16:24:32 -07:00
committed by GitHub
parent 1fe0990023
commit 9738b84a08
2 changed files with 4 additions and 29 deletions
+3 -1
View File
@@ -156,7 +156,9 @@ class PagedAttention(nn.Module):
# sequences or heads is large, we use V1 since there is enough work
# to parallelize.
# TODO(woosuk): Tune this heuristic.
use_v1 = max_num_partitions == 1 or num_seqs * num_heads > 512
# For context len > 8192, use V2 kernel to avoid shared memory shortage.
use_v1 = input_metadata.max_context_len <= 8192 and (
max_num_partitions == 1 or num_seqs * num_heads > 512)
if use_v1:
# Run PagedAttention V1.
attention_ops.paged_attention_v1(