[Bugfix][TurboQuant] Add KV quant mode for turboquant (#50533)
Signed-off-by: Soila Kavulya <soila.p.kavulya@intel.com> Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
@@ -43,6 +43,7 @@ class KVQuantMode(IntEnum):
|
||||
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
|
||||
INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp
|
||||
NVFP4 = 5 # packed fp4 data + fp8 block scales
|
||||
TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot
|
||||
|
||||
@property
|
||||
def is_per_token_head(self) -> bool:
|
||||
@@ -58,6 +59,11 @@ class KVQuantMode(IntEnum):
|
||||
"""True for NVFP4 packed quantization mode."""
|
||||
return self == KVQuantMode.NVFP4
|
||||
|
||||
@property
|
||||
def is_turboquant(self) -> bool:
|
||||
"""True for turboquant quantization mode."""
|
||||
return self == KVQuantMode.TURBOQUANT
|
||||
|
||||
|
||||
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
"""Map a ``kv_cache_dtype`` string to a :class:`KVQuantMode`."""
|
||||
@@ -69,6 +75,8 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
return KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
return KVQuantMode.NVFP4
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
||||
return KVQuantMode.TURBOQUANT
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
|
||||
return KVQuantMode.FP8_PER_TENSOR
|
||||
return KVQuantMode.NONE
|
||||
|
||||
Reference in New Issue
Block a user