Add NVFP4 KV 4-over-6 scale search (#45187)
Signed-off-by: weimingc <17592131+meenchen@users.noreply.github.com>
This commit is contained in:
@@ -528,6 +528,7 @@ def test_non_causal_autoselect_backend():
|
||||
"fp8_ds_mla",
|
||||
"fp8_inc",
|
||||
"nvfp4",
|
||||
"nvfp4_4over6",
|
||||
"fp8_per_token_head",
|
||||
"int8_per_token_head",
|
||||
],
|
||||
|
||||
@@ -426,6 +426,73 @@ def test_reshape_and_cache_flash(
|
||||
torch.testing.assert_close(value_cache_compact, cloned_value_cache)
|
||||
|
||||
|
||||
@torch.inference_mode()
|
||||
def test_nvfp4_4over6_selects_lower_error_scale(
|
||||
kv_cache_factory_flashinfer,
|
||||
) -> None:
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip("NVFP4 requires compute capability >= 10.0 (Blackwell).")
|
||||
|
||||
device = CUDA_DEVICES[0]
|
||||
block_size = 16
|
||||
head_size = 64
|
||||
source = torch.tensor(
|
||||
[6.0] + [4.5] * 15,
|
||||
dtype=torch.bfloat16,
|
||||
device=device,
|
||||
).repeat(1, 1, head_size // 16)
|
||||
slot_mapping = torch.zeros(1, dtype=torch.long, device=device)
|
||||
scale = torch.ones(1, dtype=torch.float32, device=device)
|
||||
|
||||
def quantize_and_dequantize(cache_dtype: str) -> torch.Tensor:
|
||||
key_caches, value_caches = kv_cache_factory_flashinfer(
|
||||
1,
|
||||
block_size,
|
||||
1,
|
||||
1,
|
||||
head_size,
|
||||
cache_dtype,
|
||||
source.dtype,
|
||||
device=device,
|
||||
cache_layout="NHD",
|
||||
)
|
||||
key_cache, value_cache = key_caches[0], value_caches[0]
|
||||
key_cache.zero_()
|
||||
value_cache.zero_()
|
||||
|
||||
ops.reshape_and_cache_flash(
|
||||
source,
|
||||
source,
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
cache_dtype,
|
||||
scale,
|
||||
scale,
|
||||
)
|
||||
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
dequant_nvfp4_kv_cache,
|
||||
)
|
||||
|
||||
data, block_scales = nvfp4_split_data_scale(key_cache)
|
||||
return dequant_nvfp4_kv_cache(
|
||||
data.permute(0, 2, 1, 3),
|
||||
block_scales.permute(0, 2, 1, 3),
|
||||
scale.item(),
|
||||
head_size,
|
||||
block_size,
|
||||
)[0, 0, 0]
|
||||
|
||||
default = quantize_and_dequantize("nvfp4")
|
||||
four_over_six = quantize_and_dequantize("nvfp4_4over6")
|
||||
source_f32 = source[0, 0].float()
|
||||
|
||||
default_mse = (default - source_f32).square().mean()
|
||||
four_over_six_mse = (four_over_six - source_f32).square().mean()
|
||||
assert four_over_six_mse < default_mse
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
|
||||
@pytest.mark.parametrize("kv_cache_layout", CACHE_LAYOUTS)
|
||||
|
||||
@@ -8,12 +8,26 @@ from vllm.utils.torch_utils import (
|
||||
available_cpu_count,
|
||||
common_broadcastable_dtype,
|
||||
current_stream,
|
||||
get_kv_cache_torch_dtype,
|
||||
is_lossless_cast,
|
||||
is_quantized_kv_cache,
|
||||
set_torch_threads_for_runtime,
|
||||
startup_omp_num_threads,
|
||||
)
|
||||
|
||||
|
||||
def test_nvfp4_4over6_cache_dtype() -> None:
|
||||
from vllm.config.cache import CacheConfig
|
||||
from vllm.v1.kv_cache_interface import KVQuantMode, get_kv_quant_mode
|
||||
|
||||
cache_config = CacheConfig(cache_dtype="nvfp4_4over6")
|
||||
|
||||
assert cache_config.cache_dtype == "nvfp4_4over6"
|
||||
assert get_kv_cache_torch_dtype(cache_config.cache_dtype) == torch.uint8
|
||||
assert is_quantized_kv_cache(cache_config.cache_dtype)
|
||||
assert get_kv_quant_mode(cache_config.cache_dtype) == KVQuantMode.NVFP4
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("src_dtype", "tgt_dtype", "expected_result"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user