Add fp8 support to reshape_and_cache_flash (#6667)
This commit is contained in:
+4
-1
@@ -426,10 +426,13 @@ def reshape_and_cache_flash(
|
||||
value_cache: torch.Tensor,
|
||||
slot_mapping: torch.Tensor,
|
||||
kv_cache_dtype: str,
|
||||
k_scale: float,
|
||||
v_scale: float,
|
||||
) -> None:
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash(key, value, key_cache,
|
||||
value_cache, slot_mapping,
|
||||
kv_cache_dtype)
|
||||
kv_cache_dtype, k_scale,
|
||||
v_scale)
|
||||
|
||||
|
||||
def copy_blocks(key_caches: List[torch.Tensor],
|
||||
|
||||
Reference in New Issue
Block a user