Add nvfp4 support to reshape_and_cache_flash (#37332)

Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
sychen52
2026-04-17 07:28:00 -07:00
committed by GitHub
parent 70770268c3
commit 6b2b7bd0eb
10 changed files with 679 additions and 51 deletions
+122 -12
View File
@@ -46,6 +46,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
"turboquant_4bit_nc": torch.uint8,
"turboquant_k3v4_nc": torch.uint8,
"turboquant_3bit_nc": torch.uint8,
"nvfp4": torch.uint8,
}
TORCH_DTYPE_TO_NUMPY_DTYPE = {
@@ -59,17 +60,19 @@ TORCH_DTYPE_TO_NUMPY_DTYPE = {
MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
# TODO: Add more modelopt kv cache dtype
# mappings here when it supported by some attention backend
# (for example supports nvfp4).
"fp8": "fp8_e4m3",
"nvfp4": "nvfp4",
}
T = TypeVar("T")
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
return kv_cache_dtype.startswith("fp8") or kv_cache_dtype.endswith("per_token_head")
return (
kv_cache_dtype.startswith("fp8")
or kv_cache_dtype.endswith("per_token_head")
or kv_cache_dtype == "nvfp4"
)
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
@@ -299,6 +302,8 @@ def get_kv_cache_quant_algo_string(quant_cfg: dict[str, Any]) -> str | None:
and kv_algo.get("type") == "float"
):
kv_algo = "fp8"
elif kv_algo.get("num_bits") == 4 and kv_algo.get("type") == "float":
kv_algo = "nvfp4"
else:
# Unknown/unsupported format - return "auto" as safe fallback
logger.warning(
@@ -375,6 +380,95 @@ def set_random_seed(seed: int | None) -> None:
current_platform.manual_seed_all(seed)
def nvfp4_kv_cache_full_dim(head_size: int) -> int:
"""Packed last dim for NVFP4 KV cache: fp4 data + fp8 block scales."""
return head_size // 2 + head_size // 16
def _nvfp4_split_data_scale(
kv_side: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Split a single NVFP4 KV-side buffer into data and scale views.
The input is a 4D tensor for one KV side (K or V) whose last
dimension is ``full_dim = data_dim + scale_dim``. The physical
layout within each side is [data | scale], both packed contiguously.
Args:
kv_side: 4D uint8 tensor with shape
``(num_pages, dim_1, dim_2, full_dim)``.
May be in any permutation order (NHD or HND).
Returns:
``(data, scale)`` where
``data`` is a uint8 view with shape
``(num_pages, dim_1, dim_2, data_dim)``.
``scale`` is a float8_e4m3fn view with shape
``(num_pages, dim_1, dim_2, scale_dim)``.
"""
num_pages = kv_side.shape[0]
dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2]
full_dim = kv_side.shape[3]
data_dim = full_dim * 8 // 9
scale_dim = full_dim - data_dim
data_per_kv = dim_1 * dim_2 * data_dim
page_bytes = kv_side.stride(0)
# Derive inner strides from the kv_side strides, scaling by the
# ratio of the target dim to full_dim. This preserves the physical
# layout (NHD vs HND) encoded in the input tensor's strides.
s1 = kv_side.stride(1) * data_dim // full_dim
s2 = kv_side.stride(2) * data_dim // full_dim
data_shape = (num_pages, dim_1, dim_2, data_dim)
data_strides = (page_bytes, s1, s2, 1)
s1_s = kv_side.stride(1) * scale_dim // full_dim
s2_s = kv_side.stride(2) * scale_dim // full_dim
scale_shape = (num_pages, dim_1, dim_2, scale_dim)
scale_strides = (page_bytes, s1_s, s2_s, 1)
base = kv_side.storage_offset()
data = torch.as_strided(kv_side, data_shape, data_strides, storage_offset=base)
scale = torch.as_strided(
kv_side, scale_shape, scale_strides, storage_offset=base + data_per_kv
).view(torch.float8_e4m3fn)
return data, scale
def nvfp4_kv_cache_split_views(kv_cache: torch.Tensor) -> tuple[tuple, tuple]:
"""Split an NVFP4 KV cache tensor into data and scale views.
Accepts either a 5D tensor ``(num_pages, 2, dim_2, dim_3, full_dim)``
or a 4D single-side tensor ``(num_pages, dim_2, dim_3, full_dim)``.
Per-page layout: [K_data | K_scale | V_data | V_scale].
Each KV side is self-contained (data followed by its scale), so the
5D case simply splits each side independently.
The returned views are in the same dim order as the input (NHD or
HND), so callers get views matching whichever order they passed in.
Args:
kv_cache: 5D or 4D uint8 tensor where the last dimension is
``full_dim = data_dim + scale_dim = 9 * head_size / 16``.
Returns:
For 5D input:
``(k_data, v_data), (k_scale, v_scale)``
For 4D input (single KV side):
``(data,), (scale,)``
"""
if kv_cache.dim() == 4:
data, scale = _nvfp4_split_data_scale(kv_cache)
return (data,), (scale,)
k_data, k_scale = _nvfp4_split_data_scale(kv_cache[:, 0])
v_data, v_scale = _nvfp4_split_data_scale(kv_cache[:, 1])
return (k_data, v_data), (k_scale, v_scale)
def create_kv_caches_with_random_flash(
num_blocks: int,
block_size: int,
@@ -401,15 +495,31 @@ def create_kv_caches_with_random_flash(
value_caches: list[torch.Tensor] = []
for _ in range(num_layers):
key_value_cache = torch.empty(
size=kv_cache_allocation_shape, dtype=dtype, device=device
).permute(*stride_order)
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
key_value_cache.uniform_(-scale, scale)
elif cache_dtype == "fp8":
_generate_random_fp8(key_value_cache, -scale, scale)
if cache_dtype == "nvfp4":
# Full page dim: fp4 data + fp8 block scales per head.
# Per page layout: [K_data | K_scale | V_data | V_scale]
# Returns [:, 0] and [:, 1] like all other dtypes.
full_dim = nvfp4_kv_cache_full_dim(head_size)
nvfp4_shape = (num_blocks, 2, block_size, num_heads, full_dim)
nvfp4_phys = tuple(nvfp4_shape[i] for i in stride_order)
inv = [stride_order.index(i) for i in range(len(stride_order))]
key_value_cache = torch.randint(
0,
256,
nvfp4_phys,
dtype=dtype,
device=device,
).permute(*inv)
else:
raise ValueError(f"Does not support key cache of type {cache_dtype}")
key_value_cache = torch.empty(
size=kv_cache_allocation_shape, dtype=dtype, device=device
).permute(*stride_order)
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
key_value_cache.uniform_(-scale, scale)
elif cache_dtype == "fp8":
_generate_random_fp8(key_value_cache, -scale, scale)
else:
raise ValueError(f"Does not support key cache of type {cache_dtype}")
key_caches.append(key_value_cache[:, 0])
value_caches.append(key_value_cache[:, 1])
return key_caches, value_caches