Add NVFP4 KV 4-over-6 scale search (#45187)
Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
@@ -50,6 +50,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
|
||||
"turboquant_k3v4_nc": torch.uint8,
|
||||
"turboquant_3bit_nc": torch.uint8,
|
||||
"nvfp4": torch.uint8,
|
||||
"nvfp4_4over6": torch.uint8,
|
||||
}
|
||||
|
||||
TORCH_DTYPE_TO_NUMPY_DTYPE = {
|
||||
@@ -77,7 +78,7 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return (
|
||||
kv_cache_dtype.startswith("fp8")
|
||||
or kv_cache_dtype.endswith("per_token_head")
|
||||
or kv_cache_dtype == "nvfp4"
|
||||
or kv_cache_dtype.startswith("nvfp4")
|
||||
)
|
||||
|
||||
|
||||
@@ -608,7 +609,7 @@ def create_kv_caches_with_random_flash(
|
||||
value_caches: list[torch.Tensor] = []
|
||||
|
||||
for _ in range(num_layers):
|
||||
if cache_dtype == "nvfp4":
|
||||
if isinstance(cache_dtype, str) and cache_dtype.startswith("nvfp4"):
|
||||
# Full page dim: fp4 data + fp8 block scales per head.
|
||||
# Per page layout: [K_data | K_scale | V_data | V_scale]
|
||||
# Returns [:, 0] and [:, 1] like all other dtypes.
|
||||
|
||||
Reference in New Issue
Block a user