[Bugfix][TurboQuant] Add KV quant mode for turboquant (#50533)

Signed-off-by: Soila Kavulya <soila.p.kavulya@intel.com>
Co-authored-by: Claude <noreply@anthropic.com>
This commit is contained in:
Soila Kavulya
2026-07-31 17:42:32 +00:00
committed by GitHub
co-authored by Claude
parent e67a2e0a56
commit aef85aed5d
4 changed files with 32 additions and 2 deletions
+8
View File
@@ -43,6 +43,7 @@ class KVQuantMode(IntEnum):
FP8_PER_TOKEN_HEAD = 3 # per-token-head dynamic scales for fp8
INT4_PER_TOKEN_HEAD = 4 # packed 2×int4/byte, RHT + asymmetric zp
NVFP4 = 5 # packed fp4 data + fp8 block scales
TURBOQUANT = 6 # Hadamard-rotated Lloyd-Max quant, packed K+V per slot
@property
def is_per_token_head(self) -> bool:
@@ -58,6 +59,11 @@ class KVQuantMode(IntEnum):
"""True for NVFP4 packed quantization mode."""
return self == KVQuantMode.NVFP4
@property
def is_turboquant(self) -> bool:
"""True for turboquant quantization mode."""
return self == KVQuantMode.TURBOQUANT
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
"""Map a ``kv_cache_dtype`` string to a :class:`KVQuantMode`."""
@@ -69,6 +75,8 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
return KVQuantMode.FP8_PER_TOKEN_HEAD
if kv_cache_dtype == "nvfp4":
return KVQuantMode.NVFP4
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
return KVQuantMode.TURBOQUANT
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
return KVQuantMode.FP8_PER_TENSOR
return KVQuantMode.NONE