Add NVFP4 KV 4-over-6 scale search (#45187)

Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
Wei-Ming Chen
2026-08-07 13:22:27 -07:00
committed by GitHub
parent 70456e5e6f
commit 1c94e8dc7d
11 changed files with 284 additions and 42 deletions
+3 -2
View File
@@ -50,6 +50,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
"turboquant_k3v4_nc": torch.uint8,
"turboquant_3bit_nc": torch.uint8,
"nvfp4": torch.uint8,
"nvfp4_4over6": torch.uint8,
}
TORCH_DTYPE_TO_NUMPY_DTYPE = {
@@ -77,7 +78,7 @@ def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
return (
kv_cache_dtype.startswith("fp8")
or kv_cache_dtype.endswith("per_token_head")
or kv_cache_dtype == "nvfp4"
or kv_cache_dtype.startswith("nvfp4")
)
@@ -608,7 +609,7 @@ def create_kv_caches_with_random_flash(
value_caches: list[torch.Tensor] = []
for _ in range(num_layers):
if cache_dtype == "nvfp4":
if isinstance(cache_dtype, str) and cache_dtype.startswith("nvfp4"):
# Full page dim: fp4 data + fp8 block scales per head.
# Per page layout: [K_data | K_scale | V_data | V_scale]
# Returns [:, 0] and [:, 1] like all other dtypes.