Add support for BLOOM (#331)

This commit is contained in:
Woosuk Kwon
2023-07-03 13:12:35 -07:00
committed by GitHub
parent d6fa1be3a8
commit e41f06702c
11 changed files with 479 additions and 18 deletions
+3 -1
View File
@@ -1,4 +1,5 @@
#include <torch/extension.h>
#include <c10/util/Optional.h>
void single_query_cached_kv_attention(
torch::Tensor& out,
@@ -9,7 +10,8 @@ void single_query_cached_kv_attention(
torch::Tensor& block_tables,
torch::Tensor& context_lens,
int block_size,
int max_context_len);
int max_context_len,
const c10::optional<torch::Tensor>& alibi_slopes);
PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
m.def(