[Test] Dequantize NVFP4 KV cache scales in the layout the kernel writes (#55908)
Signed-off-by: Raya Elena Solano <raya.solano@mbinf.de> Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
co-authored by
mgoin
parent
8ebc5b0a18
commit
bcca76e7db
@@ -97,8 +97,9 @@ def dequant_nvfp4_kv_cache(
|
||||
global_scale: float,
|
||||
head_size: int,
|
||||
block_size: int,
|
||||
swizzled_scales: bool = True,
|
||||
) -> torch.Tensor:
|
||||
"""Dequantize an NVFP4 KV cache with 4x4-swizzled block scales.
|
||||
"""Dequantize an NVFP4 KV cache.
|
||||
|
||||
The input must be in HND layout so that the last two dims are
|
||||
(block_size, last_dim). For NHD caches, permute to HND first.
|
||||
@@ -110,6 +111,10 @@ def dequant_nvfp4_kv_cache(
|
||||
global_scale: checkpoint dequant scale (k_scale or v_scale).
|
||||
head_size: head dimension.
|
||||
block_size: page size.
|
||||
swizzled_scales: whether ``block_scale`` is in the 4x4-swizzled
|
||||
layout of the SM100 trtllm-gen kernel. ``reshape_and_cache_flash``
|
||||
writes K scales linearly and V scales swizzled, so pass ``False``
|
||||
for K and ``True`` for V.
|
||||
|
||||
Returns:
|
||||
[..., num_heads, block_size, head_size] float32.
|
||||
@@ -118,18 +123,21 @@ def dequant_nvfp4_kv_cache(
|
||||
scale_dim = head_size // 16
|
||||
|
||||
fp4_packed = fp4_data
|
||||
sf_swizzled = block_scale.view(torch.uint8)
|
||||
|
||||
# Unswizzle 4x4 block scales on (block_size, scale_dim) plane.
|
||||
# [..., T, S] → [..., T//4, 4, sg, 4] → permute → [..., T, S]
|
||||
batch_shape = sf_swizzled.shape[:-2]
|
||||
sf_bytes = block_scale.view(torch.uint8)
|
||||
batch_shape = sf_bytes.shape[:-2]
|
||||
T, S = block_size, scale_dim
|
||||
sg = S // 4
|
||||
sf_reshape = sf_swizzled.reshape(*batch_shape, T // 4, 4, sg, 4)
|
||||
ndim = sf_reshape.ndim
|
||||
# Swap the last four dims: (..., T//4, 4, sg, 4) → (..., T//4, 4, 4, sg)
|
||||
perm = list(range(ndim - 4)) + [ndim - 4, ndim - 1, ndim - 3, ndim - 2]
|
||||
sf_linear = sf_reshape.permute(*perm).reshape(*batch_shape, T, S)
|
||||
|
||||
if swizzled_scales:
|
||||
# Unswizzle 4x4 block scales on (block_size, scale_dim) plane.
|
||||
# [..., T, S] → [..., T//4, 4, sg, 4] → permute → [..., T, S]
|
||||
sg = S // 4
|
||||
sf_reshape = sf_bytes.reshape(*batch_shape, T // 4, 4, sg, 4)
|
||||
ndim = sf_reshape.ndim
|
||||
# Swap the last four dims: (..., T//4, 4, sg, 4) → (..., T//4, 4, 4, sg)
|
||||
perm = list(range(ndim - 4)) + [ndim - 4, ndim - 1, ndim - 3, ndim - 2]
|
||||
sf_linear = sf_reshape.permute(*perm).reshape(*batch_shape, T, S)
|
||||
else:
|
||||
sf_linear = sf_bytes
|
||||
sf_f32 = sf_linear.view(torch.float8_e4m3fn).to(torch.float32)
|
||||
|
||||
# Unpack fp4
|
||||
|
||||
Reference in New Issue
Block a user