Add nvfp4 support to reshape_and_cache_flash (#37332)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
@@ -10,7 +10,7 @@ from tests.kernels.utils import DEFAULT_OPCHECK_TEST_UTILS, opcheck
|
||||
from vllm import _custom_ops as ops
|
||||
from vllm.model_executor.layers.quantization.utils.quant_utils import scaled_dequantize
|
||||
from vllm.platforms import current_platform
|
||||
from vllm.utils.torch_utils import set_random_seed
|
||||
from vllm.utils.torch_utils import nvfp4_kv_cache_split_views, set_random_seed
|
||||
|
||||
COPYING_DIRECTION = [("cuda", "cpu"), ("cuda", "cuda"), ("cpu", "cuda")]
|
||||
DTYPES = [torch.bfloat16, torch.float]
|
||||
@@ -172,7 +172,7 @@ def test_reshape_and_cache(
|
||||
@pytest.mark.parametrize("dtype", DTYPES)
|
||||
@pytest.mark.parametrize("seed", SEEDS)
|
||||
@pytest.mark.parametrize("device", CUDA_DEVICES)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE)
|
||||
@pytest.mark.parametrize("kv_cache_dtype", KV_CACHE_DTYPE + ["nvfp4"])
|
||||
@pytest.mark.parametrize("kv_cache_layout", CACHE_LAYOUTS)
|
||||
@pytest.mark.parametrize("kv_scale_type", KV_SCALE_TYPES)
|
||||
@pytest.mark.parametrize("implementation", RESHAPE_FLASH_IMPLEMENTATIONS)
|
||||
@@ -202,6 +202,25 @@ def test_reshape_and_cache_flash(
|
||||
if kv_scale_type == "attn_head" and implementation != "cuda":
|
||||
pytest.skip("Only CUDA implementation supports attn_head scaling.")
|
||||
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
if not current_platform.has_device_capability(100):
|
||||
pytest.skip("NVFP4 requires compute capability >= 10.0 (Blackwell).")
|
||||
if implementation != "cuda":
|
||||
pytest.skip("NVFP4 only supports CUDA implementation.")
|
||||
if kv_scale_type != "tensor":
|
||||
pytest.skip("NVFP4 only supports per-tensor scaling.")
|
||||
if head_size % 16 != 0:
|
||||
pytest.skip("NVFP4 requires head_size divisible by 16.")
|
||||
if (head_size // 16) % 4 != 0:
|
||||
pytest.skip(
|
||||
"NVFP4 requires (head_size // 16) divisible by 4 "
|
||||
"for 4x4 block scale swizzle."
|
||||
)
|
||||
if block_size % 4 != 0:
|
||||
pytest.skip("NVFP4 requires block_size divisible by 4.")
|
||||
if dtype not in (torch.float16, torch.bfloat16):
|
||||
pytest.skip("NVFP4 quantization only supports fp16/bf16 input.")
|
||||
|
||||
# fp8 conversion requires continugous memory buffer. Reduce the number of
|
||||
# blocks and tokens to consume less memory.
|
||||
num_tokens = num_tokens // 2
|
||||
@@ -229,7 +248,23 @@ def test_reshape_and_cache_flash(
|
||||
del key_caches
|
||||
del value_caches
|
||||
|
||||
if kv_scale_type == "tensor":
|
||||
# For nvfp4, the factory returns kv[:, 0] and kv[:, 1] like all dtypes.
|
||||
# Split views are still needed for dequant verification.
|
||||
key_scale_cache = None
|
||||
value_scale_cache = None
|
||||
nvfp4_key_data = None
|
||||
nvfp4_value_data = None
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
(nvfp4_key_data,), (key_scale_cache,) = nvfp4_kv_cache_split_views(key_cache)
|
||||
(nvfp4_value_data,), (value_scale_cache,) = nvfp4_kv_cache_split_views(
|
||||
value_cache
|
||||
)
|
||||
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
# Global scale = amax / 448 (per-tensor)
|
||||
k_scale = (key.abs().amax() / 448.0).to(torch.float32)
|
||||
v_scale = (value.abs().amax() / 448.0).to(torch.float32)
|
||||
elif kv_scale_type == "tensor":
|
||||
k_scale = (key.amax() / 64.0).to(torch.float32)
|
||||
v_scale = (value.amax() / 64.0).to(torch.float32)
|
||||
else: # "attn_head"
|
||||
@@ -240,8 +275,9 @@ def test_reshape_and_cache_flash(
|
||||
y = x if kv_cache_layout == "NHD" else x.permute(0, 2, 1, 3)
|
||||
return y.contiguous()
|
||||
|
||||
key_cache_compact = permute_and_compact(key_cache)
|
||||
value_cache_compact = permute_and_compact(value_cache)
|
||||
if kv_cache_dtype != "nvfp4":
|
||||
key_cache_compact = permute_and_compact(key_cache)
|
||||
value_cache_compact = permute_and_compact(value_cache)
|
||||
|
||||
def convert_fp8_local(output, input, scale, kv_dtype):
|
||||
fp8_input = input.view(current_platform.fp8_dtype())
|
||||
@@ -257,7 +293,7 @@ def test_reshape_and_cache_flash(
|
||||
result = fp8_input.to(output.dtype) * scale.view(1, -1, 1, 1)
|
||||
output.copy_(result)
|
||||
|
||||
# Clone the KV caches.
|
||||
# Clone the KV caches (for non-nvfp4, used as reference baseline).
|
||||
if kv_cache_dtype == "fp8":
|
||||
cloned_key_cache = torch.empty_like(key_cache_compact, dtype=torch.float16)
|
||||
convert_fp8_local(cloned_key_cache, key_cache_compact, k_scale, kv_cache_dtype)
|
||||
@@ -265,25 +301,27 @@ def test_reshape_and_cache_flash(
|
||||
convert_fp8_local(
|
||||
cloned_value_cache, value_cache_compact, v_scale, kv_cache_dtype
|
||||
)
|
||||
else:
|
||||
elif kv_cache_dtype != "nvfp4":
|
||||
cloned_key_cache = key_cache_compact.clone()
|
||||
cloned_value_cache = value_cache_compact.clone()
|
||||
|
||||
# Call the reshape_and_cache kernel.
|
||||
if implementation == "cuda":
|
||||
opcheck(
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash,
|
||||
(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
),
|
||||
cond=(head_size == HEAD_SIZES[0]),
|
||||
)
|
||||
if kv_cache_dtype != "nvfp4":
|
||||
opcheck(
|
||||
torch.ops._C_cache_ops.reshape_and_cache_flash,
|
||||
(
|
||||
key,
|
||||
value,
|
||||
key_cache,
|
||||
value_cache,
|
||||
slot_mapping,
|
||||
kv_cache_dtype,
|
||||
k_scale,
|
||||
v_scale,
|
||||
),
|
||||
cond=(head_size == HEAD_SIZES[0]),
|
||||
)
|
||||
ops.reshape_and_cache_flash(
|
||||
key,
|
||||
value,
|
||||
@@ -309,6 +347,46 @@ def test_reshape_and_cache_flash(
|
||||
k_scale,
|
||||
v_scale,
|
||||
)
|
||||
|
||||
if kv_cache_dtype == "nvfp4":
|
||||
# Verify NVFP4 by dequantizing the entire cache and comparing
|
||||
# the written positions against original bf16 values.
|
||||
# Same pattern as FP8: dequant whole cache, then extract and compare.
|
||||
from tests.kernels.quantization.nvfp4_utils import (
|
||||
dequant_nvfp4_kv_cache,
|
||||
)
|
||||
|
||||
def dequant_nvfp4_cache_nhd(data_cache, scale_cache, global_scale):
|
||||
# data_cache: [N, T, H, data_dim] NHD (contiguous inner dims)
|
||||
# scale_cache: [N, T, H, scale_dim] NHD (contiguous inner dims)
|
||||
# Permute to HND layout for the dequant utility.
|
||||
data_hnd = data_cache.permute(0, 2, 1, 3)
|
||||
scale_hnd = scale_cache.permute(0, 2, 1, 3)
|
||||
result_hnd = dequant_nvfp4_kv_cache(
|
||||
data_hnd, scale_hnd, global_scale, head_size, block_size
|
||||
)
|
||||
return result_hnd.permute(0, 2, 1, 3) # back to [N, T, H, D]
|
||||
|
||||
result_key_cache = dequant_nvfp4_cache_nhd(
|
||||
nvfp4_key_data, key_scale_cache, k_scale.item()
|
||||
)
|
||||
result_value_cache = dequant_nvfp4_cache_nhd(
|
||||
nvfp4_value_data, value_scale_cache, v_scale.item()
|
||||
)
|
||||
|
||||
# Flatten [num_blocks, block_size] → [num_slots] and index by slot_mapping.
|
||||
num_slots = num_blocks * block_size
|
||||
result_key_flat = result_key_cache.reshape(num_slots, num_heads, head_size)
|
||||
result_value_flat = result_value_cache.reshape(num_slots, num_heads, head_size)
|
||||
|
||||
torch.testing.assert_close(
|
||||
result_key_flat[slot_mapping], key.float(), atol=1.5, rtol=0.5
|
||||
)
|
||||
torch.testing.assert_close(
|
||||
result_value_flat[slot_mapping], value.float(), atol=1.5, rtol=0.5
|
||||
)
|
||||
return
|
||||
|
||||
key_cache_compact = permute_and_compact(key_cache)
|
||||
value_cache_compact = permute_and_compact(value_cache)
|
||||
|
||||
|
||||
@@ -88,6 +88,60 @@ def break_fp4_bytes(a, dtype):
|
||||
return values.reshape(m, n * 2).to(dtype=dtype)
|
||||
|
||||
|
||||
def dequant_nvfp4_kv_cache(
|
||||
fp4_data: torch.Tensor,
|
||||
block_scale: torch.Tensor,
|
||||
global_scale: float,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize an NVFP4 KV cache with 4x4-swizzled block scales.
|
||||
|
||||
The input must be in HND layout so that the last two dims are
|
||||
(block_size, last_dim). For NHD caches, permute to HND first.
|
||||
|
||||
Args:
|
||||
fp4_data: [..., num_heads, block_size, head_size//2] uint8 packed fp4.
|
||||
block_scale: [..., num_heads, block_size, head_size//16] fp8 block
|
||||
scales (as uint8 or float8_e4m3fn).
|
||||
global_scale: checkpoint dequant scale (k_scale or v_scale).
|
||||
head_size: head dimension.
|
||||
block_size: page size.
|
||||
|
||||
Returns:
|
||||
[..., num_heads, block_size, head_size] float32.
|
||||
"""
|
||||
data_dim = head_size // 2
|
||||
scale_dim = head_size // 16
|
||||
|
||||
fp4_packed = fp4_data
|
||||
sf_swizzled = block_scale.view(torch.uint8)
|
||||
|
||||
# Unswizzle 4x4 block scales on (block_size, scale_dim) plane.
|
||||
# [..., T, S] → [..., T//4, 4, sg, 4] → permute → [..., T, S]
|
||||
batch_shape = sf_swizzled.shape[:-2]
|
||||
T, S = block_size, scale_dim
|
||||
sg = S // 4
|
||||
sf_reshape = sf_swizzled.reshape(*batch_shape, T // 4, 4, sg, 4)
|
||||
ndim = sf_reshape.ndim
|
||||
# Swap the last four dims: (..., T//4, 4, sg, 4) → (..., T//4, 4, 4, sg)
|
||||
perm = list(range(ndim - 4)) + [ndim - 4, ndim - 1, ndim - 3, ndim - 2]
|
||||
sf_linear = sf_reshape.permute(*perm).reshape(*batch_shape, T, S)
|
||||
sf_f32 = sf_linear.view(torch.float8_e4m3fn).to(torch.float32)
|
||||
|
||||
# Unpack fp4
|
||||
shape = fp4_packed.shape # [..., T, data_dim]
|
||||
fp4_flat = fp4_packed.reshape(-1, data_dim)
|
||||
fp4_vals = break_fp4_bytes(fp4_flat, torch.float32)
|
||||
fp4_vals = fp4_vals.reshape(*shape[:-1], head_size)
|
||||
|
||||
# Dequant: fp4_val * block_scale * global_scale per 16-element group
|
||||
return (
|
||||
fp4_vals.reshape(*shape[:-1], scale_dim, 16)
|
||||
* (sf_f32 * global_scale).unsqueeze(-1)
|
||||
).reshape(*shape[:-1], head_size)
|
||||
|
||||
|
||||
def get_nvfp4_global_scale(a: torch.Tensor):
|
||||
return (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.abs(a).max().to(torch.float32)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user