[Misc] Add indirection layer for custom ops (#3913)

This commit is contained in:
Kunshang Ji
2024-04-10 20:26:07 -07:00
committed by GitHub
parent e42df7227d
commit e9da5a40c6
14 changed files with 224 additions and 32 deletions
+2 -2
View File
@@ -279,10 +279,10 @@ def _generate_random_fp8(
#-----|-------------|-------------------
# Inf | N/A | s.11111.00
# NaN | s.1111.111 | s.11111.{01,10,11}
from vllm._C import cache_ops
from vllm import _custom_ops as ops
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
tensor_tmp.uniform_(low, high)
cache_ops.convert_fp8(tensor_tmp, tensor)
ops.convert_fp8(tensor_tmp, tensor)
del tensor_tmp