Support FP8-E5M2 KV Cache (#2279)
Co-authored-by: zhaoyang <zhao.yang16@zte.com.cn> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
This commit is contained in:
co-authored by
zhaoyang
Zhuohan Li
parent
7d648418b8
commit
9090bf02e7
+109
-1
@@ -1,9 +1,11 @@
|
||||
import enum
|
||||
import os
|
||||
import socket
|
||||
import subprocess
|
||||
import uuid
|
||||
from platform import uname
|
||||
from typing import List
|
||||
from typing import List, Tuple, Union
|
||||
from packaging.version import parse, Version
|
||||
|
||||
import psutil
|
||||
import torch
|
||||
@@ -17,7 +19,17 @@ from typing import (
|
||||
from collections import OrderedDict
|
||||
from typing import Any, Hashable, Optional
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
T = TypeVar("T")
|
||||
logger = init_logger(__name__)
|
||||
|
||||
STR_DTYPE_TO_TORCH_DTYPE = {
|
||||
"half": torch.half,
|
||||
"bfloat16": torch.bfloat16,
|
||||
"float": torch.float,
|
||||
"fp8_e5m2": torch.uint8,
|
||||
}
|
||||
|
||||
|
||||
class Device(enum.Enum):
|
||||
@@ -167,3 +179,99 @@ def get_open_port() -> int:
|
||||
|
||||
def set_cuda_visible_devices(device_ids: List[int]) -> None:
|
||||
os.environ["CUDA_VISIBLE_DEVICES"] = ",".join(map(str, device_ids))
|
||||
|
||||
|
||||
def get_nvcc_cuda_version() -> Version:
|
||||
cuda_home = os.environ.get('CUDA_HOME')
|
||||
if not cuda_home:
|
||||
cuda_home = '/usr/local/cuda'
|
||||
logger.info(
|
||||
f'CUDA_HOME is not found in the environment. Using {cuda_home} as CUDA_HOME.'
|
||||
)
|
||||
nvcc_output = subprocess.check_output([cuda_home + "/bin/nvcc", "-V"],
|
||||
universal_newlines=True)
|
||||
output = nvcc_output.split()
|
||||
release_idx = output.index("release") + 1
|
||||
nvcc_cuda_version = parse(output[release_idx].split(",")[0])
|
||||
return nvcc_cuda_version
|
||||
|
||||
|
||||
def _generate_random_fp8_e5m2(
|
||||
tensor: torch.tensor,
|
||||
low: float,
|
||||
high: float,
|
||||
) -> None:
|
||||
# NOTE(zhaoyang): Due to NaN and Inf representation for fp8 data type,
|
||||
# it may occur Inf or NaN if we directly use torch.randint
|
||||
# to generate random data for fp8 data.
|
||||
# For example, s.11111.00 in fp8e5m2 format repesents Inf.
|
||||
# | E4M3 | E5M2
|
||||
#-----|-------------|-------------------
|
||||
# Inf | N/A | s.11111.00
|
||||
# NaN | s.1111.111 | s.11111.{01,10,11}
|
||||
from vllm._C import cache_ops
|
||||
tensor_tmp = torch.empty_like(tensor, dtype=torch.float16)
|
||||
tensor_tmp.uniform_(low, high)
|
||||
cache_ops.convert_fp8_e5m2(tensor_tmp, tensor)
|
||||
del tensor_tmp
|
||||
|
||||
|
||||
def create_kv_caches_with_random(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_layers: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype: Optional[Union[str, torch.dtype]],
|
||||
model_dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
seed: Optional[int] = 0,
|
||||
device: Optional[str] = "cuda",
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
torch.random.manual_seed(seed)
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
if isinstance(cache_dtype, str):
|
||||
if cache_dtype == "auto":
|
||||
if isinstance(model_dtype, str):
|
||||
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[model_dtype]
|
||||
elif isinstance(model_dtype, torch.dtype):
|
||||
torch_dtype = model_dtype
|
||||
else:
|
||||
raise ValueError(f"Invalid model dtype: {model_dtype}")
|
||||
elif cache_dtype in ["half", "bfloat16", "float"]:
|
||||
torch_dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_dtype]
|
||||
elif cache_dtype == "fp8_e5m2":
|
||||
torch_dtype = torch.uint8
|
||||
else:
|
||||
raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
|
||||
elif isinstance(cache_dtype, torch.dtype):
|
||||
torch_dtype = cache_dtype
|
||||
else:
|
||||
raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
|
||||
|
||||
scale = head_size**-0.5
|
||||
x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
|
||||
key_cache_shape = (num_blocks, num_heads, head_size // x, block_size, x)
|
||||
key_caches = []
|
||||
for _ in range(num_layers):
|
||||
key_cache = torch.empty(size=key_cache_shape,
|
||||
dtype=torch_dtype,
|
||||
device=device)
|
||||
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
|
||||
key_cache.uniform_(-scale, scale)
|
||||
elif cache_dtype == 'fp8_e5m2':
|
||||
_generate_random_fp8_e5m2(key_cache, -scale, scale)
|
||||
key_caches.append(key_cache)
|
||||
|
||||
value_cache_shape = (num_blocks, num_heads, head_size, block_size)
|
||||
value_caches = []
|
||||
for _ in range(num_layers):
|
||||
value_cache = torch.empty(size=value_cache_shape,
|
||||
dtype=torch_dtype,
|
||||
device=device)
|
||||
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
|
||||
value_cache.uniform_(-scale, scale)
|
||||
elif cache_dtype == 'fp8_e5m2':
|
||||
_generate_random_fp8_e5m2(value_cache, -scale, scale)
|
||||
value_caches.append(value_cache)
|
||||
return key_caches, value_caches
|
||||
|
||||
Reference in New Issue
Block a user