[Kernel][Attention] Separate Attention.kv_scale into k_scale and v_scale (#6081)

This commit is contained in:
Michael Goin
2024-07-16 15:31:32 -07:00
committed by GitHub
parent 160e1d8c99
commit 978aed5300
33 changed files with 317 additions and 185 deletions
+5 -3
View File
@@ -175,7 +175,7 @@ def test_paged_attention(
key_cache, value_cache = key_caches[0], value_caches[0]
# Using default kv_scale
kv_scale = 1.0
k_scale = v_scale = 1.0
# Call the paged attention kernel.
output = torch.empty_like(query)
@@ -193,7 +193,8 @@ def test_paged_attention(
max_seq_len,
alibi_slopes,
kv_cache_dtype,
kv_scale,
k_scale,
v_scale,
)
elif version == "v2":
num_partitions = ((max_seq_len + PARTITION_SIZE - 1) // PARTITION_SIZE)
@@ -224,7 +225,8 @@ def test_paged_attention(
max_seq_len,
alibi_slopes,
kv_cache_dtype,
kv_scale,
k_scale,
v_scale,
)
else:
raise AssertionError(f"Unknown version: {version}")