[Feature] KV cache per-token-head INT8/FP8 quantization (#38378)
Signed-off-by: JartX <sagformas@epdcenter.es> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: yangyang4991 <yangyang4991@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Isotr0py <2037008807@qq.com>
This commit is contained in:
co-authored by
yangyang4991
Wentao Ye
Isotr0py
parent
4eefbf9609
commit
2ce3d0ce36
@@ -37,6 +37,8 @@ STR_DTYPE_TO_TORCH_DTYPE = {
|
||||
"fp8_e4m3": torch.uint8,
|
||||
"fp8_e5m2": torch.uint8,
|
||||
"int8": torch.int8,
|
||||
"int8_per_token_head": torch.int8,
|
||||
"fp8_per_token_head": torch.uint8,
|
||||
"fp8_inc": torch.float8_e4m3fn,
|
||||
"fp8_ds_mla": torch.uint8,
|
||||
}
|
||||
@@ -62,7 +64,12 @@ T = TypeVar("T")
|
||||
|
||||
|
||||
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return kv_cache_dtype.startswith("fp8")
|
||||
return kv_cache_dtype.startswith("fp8") or kv_cache_dtype.endswith("per_token_head")
|
||||
|
||||
|
||||
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
|
||||
"""Return True if *kv_cache_dtype* needs per-token-head scales."""
|
||||
return kv_cache_dtype.endswith("per_token_head")
|
||||
|
||||
|
||||
def is_strictly_contiguous(t: torch.Tensor) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user