[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)

Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
aditi-amd
2026-10-06 20:42:52 +00:00
committed by GitHub
parent 819852df6b
commit 385d86a6cf
22 changed files with 5377 additions and 24 deletions
+8
View File
@@ -55,6 +55,7 @@ class KVQuantMode(IntEnum):
TURBOQUANT_K3V4_NC = 8
TURBOQUANT_3BIT_NC = 9
NVFP4_DS_MLA = 10 # opaque-bytes NVFP4 DS-MLA layouts (FlashMLA sparse)
ULTRAQUANT_4BIT = 11 # packed FP4 values + UE8M0 group scales
@property
def is_per_token_head(self) -> bool:
@@ -80,6 +81,11 @@ class KVQuantMode(IntEnum):
KVQuantMode.TURBOQUANT_3BIT_NC,
)
@property
def is_ultraquant(self) -> bool:
"""True for the UltraQuant 4-bit KV-cache format."""
return self == KVQuantMode.ULTRAQUANT_4BIT
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
"""Map a ``kv_cache_dtype`` string to a :class:`KVQuantMode`."""
@@ -98,6 +104,8 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
return KVQuantMode.NVFP4
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
return KVQuantMode[kv_cache_dtype.upper()]
if kv_cache_dtype == "ultraquant_4bit":
return KVQuantMode.ULTRAQUANT_4BIT
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
return KVQuantMode.FP8_PER_TENSOR
return KVQuantMode.NONE