[Bugfix] Add kv_scale input parameter to CPU backend (#3840)

This commit is contained in:
Woosuk Kwon
2024-04-04 04:33:08 +00:00
committed by GitHub
parent 537ee25f43
commit 498eb5cfa3
4 changed files with 12 additions and 5 deletions
+4 -1
View File
@@ -114,6 +114,7 @@ class TorchSDPABackendImpl(AttentionImpl):
value: torch.Tensor,
kv_cache: Optional[torch.Tensor],
attn_metadata: TorchSDPAMetadata,
kv_scale: float,
) -> torch.Tensor:
"""Forward pass with torch SDPA and PagedAttention.
@@ -138,7 +139,8 @@ class TorchSDPABackendImpl(AttentionImpl):
PagedAttention.write_to_paged_cache(key, value, key_cache,
value_cache,
attn_metadata.slot_mapping,
attn_metadata.kv_cache_dtype)
attn_metadata.kv_cache_dtype,
kv_scale)
if attn_metadata.is_prompt:
if (kv_cache is None or attn_metadata.block_tables.numel() == 0):
@@ -199,6 +201,7 @@ class TorchSDPABackendImpl(AttentionImpl):
self.num_kv_heads,
self.scale,
self.alibi_slopes,
kv_scale,
)
# Reshape the output tensor.
+1 -1
View File
@@ -97,7 +97,7 @@ class PagedAttention:
num_kv_heads: int,
scale: float,
alibi_slopes: Optional[torch.Tensor],
kv_scale,
kv_scale: float,
) -> torch.Tensor:
output = torch.empty_like(query)