[Kernel] Use flashinfer for decoding (#4353)
Co-authored-by: LiuXiaoxuanPKU <llilyliupku@gmail.com>
This commit is contained in:
co-authored by
LiuXiaoxuanPKU
parent
f8e7adda21
commit
43c413ec57
+52
-15
@@ -355,21 +355,9 @@ def _generate_random_fp8(
|
||||
del tensor_tmp
|
||||
|
||||
|
||||
def create_kv_caches_with_random(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_layers: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype: Optional[Union[str, torch.dtype]],
|
||||
model_dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
seed: int = 0,
|
||||
device: Optional[str] = "cuda",
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
torch.random.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
def get_kv_cache_torch_dtype(
|
||||
cache_dtype: Optional[Union[str, torch.dtype]],
|
||||
model_dtype: Optional[Union[str, torch.dtype]] = None) -> torch.dtype:
|
||||
if isinstance(cache_dtype, str):
|
||||
if cache_dtype == "auto":
|
||||
if isinstance(model_dtype, str):
|
||||
@@ -388,6 +376,55 @@ def create_kv_caches_with_random(
|
||||
torch_dtype = cache_dtype
|
||||
else:
|
||||
raise ValueError(f"Invalid kv cache dtype: {cache_dtype}")
|
||||
return torch_dtype
|
||||
|
||||
|
||||
def create_kv_caches_with_random_flash(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_layers: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype: Optional[Union[str, torch.dtype]],
|
||||
model_dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
seed: int = 0,
|
||||
device: Optional[str] = "cuda",
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
assert cache_dtype != "fp8"
|
||||
torch.random.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
|
||||
key_value_cache_shape = (num_blocks, 2, block_size, num_heads, head_size)
|
||||
scale = head_size**-0.5
|
||||
key_caches, value_caches = [], []
|
||||
for _ in range(num_layers):
|
||||
key_value_cache = torch.empty(size=key_value_cache_shape,
|
||||
dtype=torch_dtype,
|
||||
device=device)
|
||||
key_value_cache.uniform_(-scale, scale)
|
||||
key_caches.append(key_value_cache[:, 0])
|
||||
value_caches.append(key_value_cache[:, 1])
|
||||
return key_caches, value_caches
|
||||
|
||||
|
||||
def create_kv_caches_with_random(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
num_layers: int,
|
||||
num_heads: int,
|
||||
head_size: int,
|
||||
cache_dtype: Optional[Union[str, torch.dtype]],
|
||||
model_dtype: Optional[Union[str, torch.dtype]] = None,
|
||||
seed: int = 0,
|
||||
device: Optional[str] = "cuda",
|
||||
) -> Tuple[List[torch.Tensor], List[torch.Tensor]]:
|
||||
torch.random.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed(seed)
|
||||
|
||||
torch_dtype = get_kv_cache_torch_dtype(cache_dtype, model_dtype)
|
||||
|
||||
scale = head_size**-0.5
|
||||
x = 16 // torch.tensor([], dtype=torch_dtype).element_size()
|
||||
|
||||
Reference in New Issue
Block a user