Implement custom kernel for LLaMA rotary embedding (#14)

This commit is contained in:
Woosuk Kwon
2023-03-30 11:04:21 -07:00
committed by GitHub
parent 80a2f812f1
commit 88c0268a18
10 changed files with 318 additions and 69 deletions
+8
View File
@@ -23,6 +23,14 @@ attention_extension = cpp_extension.CUDAExtension(
)
ext_modules.append(attention_extension)
# Positional encodings.
positional_encoding_extension = cpp_extension.CUDAExtension(
name='cacheflow.pos_encoding_ops',
sources=['csrc/pos_encoding.cpp', 'csrc/pos_encoding_kernels.cu'],
extra_compile_args={'cxx': CXX_FLAGS, 'nvcc': NVCC_FLAGS},
)
ext_modules.append(positional_encoding_extension)
setuptools.setup(
name='cacheflow',
ext_modules=ext_modules,