[Kernel] Refactor FP8 kv-cache with NVIDIA float8_e4m3 support (#4535)

This commit is contained in:
Cody Yu
2024-05-09 18:04:17 -06:00
committed by GitHub
parent 379da6dcb5
commit c833101740
17 changed files with 843 additions and 558 deletions
+1 -1
View File
@@ -329,7 +329,7 @@ def _generate_random_fp8(
from vllm import _custom_ops as ops
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
tensor_tmp.uniform_(low, high)
ops.convert_fp8(tensor_tmp, tensor)
ops.convert_fp8(tensor, tensor_tmp)
del tensor_tmp