Use FlashAttention for multi_query_kv_attention (#4)
This commit is contained in:
@@ -9,10 +9,12 @@ parser = argparse.ArgumentParser(description='CacheFlow server')
|
||||
parser.add_argument('--model', type=str, default='facebook/opt-125m', help='model name')
|
||||
parser.add_argument('--num-nodes', type=int, default=1, help='number of nodes')
|
||||
parser.add_argument('--num-workers', type=int, default=1, help='number of workers per node')
|
||||
parser.add_argument('--block-size', type=int, default=8, help='token block size')
|
||||
parser.add_argument('--block-size', type=int, default=8, choices=[8, 16], help='token block size')
|
||||
# TODO(woosuk): Add an analytical model to determine the maximum number of GPU/CPU blocks.
|
||||
parser.add_argument('--num-gpu-blocks', type=int, default=1024, help='number of GPU blocks (per GPU)')
|
||||
parser.add_argument('--num-cpu-blocks', type=int, default=256, help='number of CPU blocks (per GPU)')
|
||||
parser.add_argument('--num-cpu-blocks', type=int, default=32, help='number of CPU blocks (per GPU)')
|
||||
# NOTE(woosuk): If FlashAttention is used, the float data type is not supported.
|
||||
parser.add_argument('--dtype', type=str, default='half', choices=['half', 'float'], help='data type')
|
||||
args = parser.parse_args()
|
||||
|
||||
|
||||
@@ -27,6 +29,7 @@ def main():
|
||||
block_size=args.block_size,
|
||||
num_gpu_blocks=args.num_gpu_blocks,
|
||||
num_cpu_blocks=args.num_cpu_blocks,
|
||||
dtype=args.dtype,
|
||||
)
|
||||
controllers.append(controller)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user