Add fp8 support to reshape_and_cache_flash (#6667)

This commit is contained in:
Antoni Baum
2024-07-24 18:36:52 +00:00
committed by GitHub
parent ee812580f7
commit 0e63494cf3
8 changed files with 98 additions and 43 deletions
+2 -1
View File
@@ -25,7 +25,8 @@ void reshape_and_cache_flash(torch::Tensor& key, torch::Tensor& value,
torch::Tensor& key_cache,
torch::Tensor& value_cache,
torch::Tensor& slot_mapping,
const std::string& kv_cache_dtype);
const std::string& kv_cache_dtype,
const double k_scale, const double v_scale);
// Just for unittest
void convert_fp8(torch::Tensor& dst_cache, torch::Tensor& src_cache,