Fix config for Falcon (#1164)

This commit is contained in:
Woosuk Kwon
2023-09-23 17:38:43 -07:00
committed by GitHub
parent f187877945
commit 9f6be8692e
2 changed files with 8 additions and 3 deletions
+2 -2
View File
@@ -33,7 +33,7 @@ class CacheEngine:
self.head_size = model_config.get_head_size()
self.num_layers = model_config.get_num_layers(parallel_config)
self.num_heads = model_config.get_num_heads(parallel_config)
self.num_heads = model_config.get_num_kv_heads(parallel_config)
self.dtype = model_config.dtype
self.block_size = cache_config.block_size
@@ -146,7 +146,7 @@ class CacheEngine:
parallel_config: ParallelConfig,
) -> int:
head_size = model_config.get_head_size()
num_heads = model_config.get_num_heads(parallel_config)
num_heads = model_config.get_num_kv_heads(parallel_config)
num_layers = model_config.get_num_layers(parallel_config)
key_cache_block = block_size * num_heads * head_size