[Attention] Add UltraQuant 4-bit KV cache backend (FlyDSL D=256) (#57057)
Signed-off-by: Aditi Rana <aditi.rana@amd.com>
This commit is contained in:
@@ -55,6 +55,7 @@ class KVQuantMode(IntEnum):
|
||||
TURBOQUANT_K3V4_NC = 8
|
||||
TURBOQUANT_3BIT_NC = 9
|
||||
NVFP4_DS_MLA = 10 # opaque-bytes NVFP4 DS-MLA layouts (FlashMLA sparse)
|
||||
ULTRAQUANT_4BIT = 11 # packed FP4 values + UE8M0 group scales
|
||||
|
||||
@property
|
||||
def is_per_token_head(self) -> bool:
|
||||
@@ -80,6 +81,11 @@ class KVQuantMode(IntEnum):
|
||||
KVQuantMode.TURBOQUANT_3BIT_NC,
|
||||
)
|
||||
|
||||
@property
|
||||
def is_ultraquant(self) -> bool:
|
||||
"""True for the UltraQuant 4-bit KV-cache format."""
|
||||
return self == KVQuantMode.ULTRAQUANT_4BIT
|
||||
|
||||
|
||||
def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
"""Map a ``kv_cache_dtype`` string to a :class:`KVQuantMode`."""
|
||||
@@ -98,6 +104,8 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
return KVQuantMode.NVFP4
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
||||
return KVQuantMode[kv_cache_dtype.upper()]
|
||||
if kv_cache_dtype == "ultraquant_4bit":
|
||||
return KVQuantMode.ULTRAQUANT_4BIT
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("fp8"):
|
||||
return KVQuantMode.FP8_PER_TENSOR
|
||||
return KVQuantMode.NONE
|
||||
|
||||
Reference in New Issue
Block a user