Add nvfp4 support to reshape_and_cache_flash (#37332)
Signed-off-by: Shiyang Chen <shiychen@nvidia.com>
This commit is contained in:
+122
-12
@@ -46,6 +46,7 @@ STR_DTYPE_TO_TORCH_DTYPE = {
|
||||
"turboquant_4bit_nc": torch.uint8,
|
||||
"turboquant_k3v4_nc": torch.uint8,
|
||||
"turboquant_3bit_nc": torch.uint8,
|
||||
"nvfp4": torch.uint8,
|
||||
}
|
||||
|
||||
TORCH_DTYPE_TO_NUMPY_DTYPE = {
|
||||
@@ -59,17 +60,19 @@ TORCH_DTYPE_TO_NUMPY_DTYPE = {
|
||||
|
||||
|
||||
MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP = {
|
||||
# TODO: Add more modelopt kv cache dtype
|
||||
# mappings here when it supported by some attention backend
|
||||
# (for example supports nvfp4).
|
||||
"fp8": "fp8_e4m3",
|
||||
"nvfp4": "nvfp4",
|
||||
}
|
||||
|
||||
T = TypeVar("T")
|
||||
|
||||
|
||||
def is_quantized_kv_cache(kv_cache_dtype: str) -> bool:
|
||||
return kv_cache_dtype.startswith("fp8") or kv_cache_dtype.endswith("per_token_head")
|
||||
return (
|
||||
kv_cache_dtype.startswith("fp8")
|
||||
or kv_cache_dtype.endswith("per_token_head")
|
||||
or kv_cache_dtype == "nvfp4"
|
||||
)
|
||||
|
||||
|
||||
def kv_cache_uses_per_token_head_scales(kv_cache_dtype: str) -> bool:
|
||||
@@ -299,6 +302,8 @@ def get_kv_cache_quant_algo_string(quant_cfg: dict[str, Any]) -> str | None:
|
||||
and kv_algo.get("type") == "float"
|
||||
):
|
||||
kv_algo = "fp8"
|
||||
elif kv_algo.get("num_bits") == 4 and kv_algo.get("type") == "float":
|
||||
kv_algo = "nvfp4"
|
||||
else:
|
||||
# Unknown/unsupported format - return "auto" as safe fallback
|
||||
logger.warning(
|
||||
@@ -375,6 +380,95 @@ def set_random_seed(seed: int | None) -> None:
|
||||
current_platform.manual_seed_all(seed)
|
||||
|
||||
|
||||
def nvfp4_kv_cache_full_dim(head_size: int) -> int:
|
||||
"""Packed last dim for NVFP4 KV cache: fp4 data + fp8 block scales."""
|
||||
return head_size // 2 + head_size // 16
|
||||
|
||||
|
||||
def _nvfp4_split_data_scale(
|
||||
kv_side: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Split a single NVFP4 KV-side buffer into data and scale views.
|
||||
|
||||
The input is a 4D tensor for one KV side (K or V) whose last
|
||||
dimension is ``full_dim = data_dim + scale_dim``. The physical
|
||||
layout within each side is [data | scale], both packed contiguously.
|
||||
|
||||
Args:
|
||||
kv_side: 4D uint8 tensor with shape
|
||||
``(num_pages, dim_1, dim_2, full_dim)``.
|
||||
May be in any permutation order (NHD or HND).
|
||||
|
||||
Returns:
|
||||
``(data, scale)`` where
|
||||
``data`` is a uint8 view with shape
|
||||
``(num_pages, dim_1, dim_2, data_dim)``.
|
||||
``scale`` is a float8_e4m3fn view with shape
|
||||
``(num_pages, dim_1, dim_2, scale_dim)``.
|
||||
"""
|
||||
num_pages = kv_side.shape[0]
|
||||
dim_1, dim_2 = kv_side.shape[1], kv_side.shape[2]
|
||||
full_dim = kv_side.shape[3]
|
||||
data_dim = full_dim * 8 // 9
|
||||
scale_dim = full_dim - data_dim
|
||||
|
||||
data_per_kv = dim_1 * dim_2 * data_dim
|
||||
page_bytes = kv_side.stride(0)
|
||||
|
||||
# Derive inner strides from the kv_side strides, scaling by the
|
||||
# ratio of the target dim to full_dim. This preserves the physical
|
||||
# layout (NHD vs HND) encoded in the input tensor's strides.
|
||||
s1 = kv_side.stride(1) * data_dim // full_dim
|
||||
s2 = kv_side.stride(2) * data_dim // full_dim
|
||||
data_shape = (num_pages, dim_1, dim_2, data_dim)
|
||||
data_strides = (page_bytes, s1, s2, 1)
|
||||
|
||||
s1_s = kv_side.stride(1) * scale_dim // full_dim
|
||||
s2_s = kv_side.stride(2) * scale_dim // full_dim
|
||||
scale_shape = (num_pages, dim_1, dim_2, scale_dim)
|
||||
scale_strides = (page_bytes, s1_s, s2_s, 1)
|
||||
|
||||
base = kv_side.storage_offset()
|
||||
data = torch.as_strided(kv_side, data_shape, data_strides, storage_offset=base)
|
||||
scale = torch.as_strided(
|
||||
kv_side, scale_shape, scale_strides, storage_offset=base + data_per_kv
|
||||
).view(torch.float8_e4m3fn)
|
||||
|
||||
return data, scale
|
||||
|
||||
|
||||
def nvfp4_kv_cache_split_views(kv_cache: torch.Tensor) -> tuple[tuple, tuple]:
|
||||
"""Split an NVFP4 KV cache tensor into data and scale views.
|
||||
|
||||
Accepts either a 5D tensor ``(num_pages, 2, dim_2, dim_3, full_dim)``
|
||||
or a 4D single-side tensor ``(num_pages, dim_2, dim_3, full_dim)``.
|
||||
|
||||
Per-page layout: [K_data | K_scale | V_data | V_scale].
|
||||
Each KV side is self-contained (data followed by its scale), so the
|
||||
5D case simply splits each side independently.
|
||||
|
||||
The returned views are in the same dim order as the input (NHD or
|
||||
HND), so callers get views matching whichever order they passed in.
|
||||
|
||||
Args:
|
||||
kv_cache: 5D or 4D uint8 tensor where the last dimension is
|
||||
``full_dim = data_dim + scale_dim = 9 * head_size / 16``.
|
||||
|
||||
Returns:
|
||||
For 5D input:
|
||||
``(k_data, v_data), (k_scale, v_scale)``
|
||||
For 4D input (single KV side):
|
||||
``(data,), (scale,)``
|
||||
"""
|
||||
if kv_cache.dim() == 4:
|
||||
data, scale = _nvfp4_split_data_scale(kv_cache)
|
||||
return (data,), (scale,)
|
||||
|
||||
k_data, k_scale = _nvfp4_split_data_scale(kv_cache[:, 0])
|
||||
v_data, v_scale = _nvfp4_split_data_scale(kv_cache[:, 1])
|
||||
return (k_data, v_data), (k_scale, v_scale)
|
||||
|
||||
|
||||
def create_kv_caches_with_random_flash(
|
||||
num_blocks: int,
|
||||
block_size: int,
|
||||
@@ -401,15 +495,31 @@ def create_kv_caches_with_random_flash(
|
||||
value_caches: list[torch.Tensor] = []
|
||||
|
||||
for _ in range(num_layers):
|
||||
key_value_cache = torch.empty(
|
||||
size=kv_cache_allocation_shape, dtype=dtype, device=device
|
||||
).permute(*stride_order)
|
||||
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
|
||||
key_value_cache.uniform_(-scale, scale)
|
||||
elif cache_dtype == "fp8":
|
||||
_generate_random_fp8(key_value_cache, -scale, scale)
|
||||
if cache_dtype == "nvfp4":
|
||||
# Full page dim: fp4 data + fp8 block scales per head.
|
||||
# Per page layout: [K_data | K_scale | V_data | V_scale]
|
||||
# Returns [:, 0] and [:, 1] like all other dtypes.
|
||||
full_dim = nvfp4_kv_cache_full_dim(head_size)
|
||||
nvfp4_shape = (num_blocks, 2, block_size, num_heads, full_dim)
|
||||
nvfp4_phys = tuple(nvfp4_shape[i] for i in stride_order)
|
||||
inv = [stride_order.index(i) for i in range(len(stride_order))]
|
||||
key_value_cache = torch.randint(
|
||||
0,
|
||||
256,
|
||||
nvfp4_phys,
|
||||
dtype=dtype,
|
||||
device=device,
|
||||
).permute(*inv)
|
||||
else:
|
||||
raise ValueError(f"Does not support key cache of type {cache_dtype}")
|
||||
key_value_cache = torch.empty(
|
||||
size=kv_cache_allocation_shape, dtype=dtype, device=device
|
||||
).permute(*stride_order)
|
||||
if cache_dtype in ["auto", "half", "bfloat16", "float"]:
|
||||
key_value_cache.uniform_(-scale, scale)
|
||||
elif cache_dtype == "fp8":
|
||||
_generate_random_fp8(key_value_cache, -scale, scale)
|
||||
else:
|
||||
raise ValueError(f"Does not support key cache of type {cache_dtype}")
|
||||
key_caches.append(key_value_cache[:, 0])
|
||||
value_caches.append(key_value_cache[:, 1])
|
||||
return key_caches, value_caches
|
||||
|
||||
Reference in New Issue
Block a user