[Kernel] Refactor FP8 kv-cache with NVIDIA float8_e4m3 support (#4535)

This commit is contained in:
Cody Yu
2024-05-09 18:04:17 -06:00
committed by GitHub
parent 379da6dcb5
commit c833101740
17 changed files with 843 additions and 558 deletions
+5 -2
View File
@@ -270,8 +270,11 @@ def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
vllm_cache_ops.swap_blocks(src, dst, block_mapping)
def convert_fp8(output: torch.Tensor, input: torch.Tensor) -> None:
vllm_cache_ops.convert_fp8(output, input)
def convert_fp8(output: torch.Tensor,
input: torch.Tensor,
scale: float = 1.0,
kv_dtype: str = "fp8") -> None:
vllm_cache_ops.convert_fp8(output, input, scale, kv_dtype)
#TODO: cuda_utils, custom_ar
+1 -1
View File
@@ -329,7 +329,7 @@ def _generate_random_fp8(
from vllm import _custom_ops as ops
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
tensor_tmp.uniform_(low, high)
ops.convert_fp8(tensor_tmp, tensor)
ops.convert_fp8(tensor, tensor_tmp)
del tensor_tmp