Add NVFP4 KV 4-over-6 scale search (#45187)

Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
Wei-Ming Chen
2026-08-07 13:22:27 -07:00
committed by GitHub
parent 70456e5e6f
commit 1c94e8dc7d
11 changed files with 284 additions and 42 deletions
+14
View File
@@ -8,12 +8,26 @@ from vllm.utils.torch_utils import (
available_cpu_count,
common_broadcastable_dtype,
current_stream,
get_kv_cache_torch_dtype,
is_lossless_cast,
is_quantized_kv_cache,
set_torch_threads_for_runtime,
startup_omp_num_threads,
)
def test_nvfp4_4over6_cache_dtype() -> None:
from vllm.config.cache import CacheConfig
from vllm.v1.kv_cache_interface import KVQuantMode, get_kv_quant_mode
cache_config = CacheConfig(cache_dtype="nvfp4_4over6")
assert cache_config.cache_dtype == "nvfp4_4over6"
assert get_kv_cache_torch_dtype(cache_config.cache_dtype) == torch.uint8
assert is_quantized_kv_cache(cache_config.cache_dtype)
assert get_kv_quant_mode(cache_config.cache_dtype) == KVQuantMode.NVFP4
@pytest.mark.parametrize(
("src_dtype", "tgt_dtype", "expected_result"),
[