Add NVFP4 KV 4-over-6 scale search (#45187)
Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
@@ -73,7 +73,7 @@ def get_kv_quant_mode(kv_cache_dtype: str) -> KVQuantMode:
|
||||
return KVQuantMode.INT8_PER_TOKEN_HEAD
|
||||
if kv_cache_dtype == "fp8_per_token_head":
|
||||
return KVQuantMode.FP8_PER_TOKEN_HEAD
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
if kv_cache_dtype.startswith("nvfp4"):
|
||||
return KVQuantMode.NVFP4
|
||||
if isinstance(kv_cache_dtype, str) and kv_cache_dtype.startswith("turboquant_"):
|
||||
return KVQuantMode.TURBOQUANT
|
||||
|
||||
Reference in New Issue
Block a user