[Kernel] Refactor FP8 kv-cache with NVIDIA float8_e4m3 support (#4535)
This commit is contained in:
+5
-2
@@ -270,8 +270,11 @@ def swap_blocks(src: torch.Tensor, dst: torch.Tensor,
|
||||
vllm_cache_ops.swap_blocks(src, dst, block_mapping)
|
||||
|
||||
|
||||
def convert_fp8(output: torch.Tensor, input: torch.Tensor) -> None:
|
||||
vllm_cache_ops.convert_fp8(output, input)
|
||||
def convert_fp8(output: torch.Tensor,
|
||||
input: torch.Tensor,
|
||||
scale: float = 1.0,
|
||||
kv_dtype: str = "fp8") -> None:
|
||||
vllm_cache_ops.convert_fp8(output, input, scale, kv_dtype)
|
||||
|
||||
|
||||
#TODO: cuda_utils, custom_ar
|
||||
|
||||
+1
-1
@@ -329,7 +329,7 @@ def _generate_random_fp8(
|
||||
from vllm import _custom_ops as ops
|
||||
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
|
||||
tensor_tmp.uniform_(low, high)
|
||||
ops.convert_fp8(tensor_tmp, tensor)
|
||||
ops.convert_fp8(tensor, tensor_tmp)
|
||||
del tensor_tmp
|
||||
|
||||
|
||||
|
||||
Reference in New Issue
Block a user