[Kernel] Add b12x FP4 MoE backend (#52018)

Signed-off-by: Luke Alonso <lalonso@gmail.com>
Signed-off-by: mgoin <mgoin64@gmail.com>
Signed-off-by: Misha Goin <mgoin64@gmail.com>
Co-authored-by: mgoin <mgoin64@gmail.com>
This commit is contained in:
Luke Alonso
2026-08-21 08:04:36 -07:00
committed by GitHub
co-authored by mgoin
parent 5ee84d3c52
commit 2740c817ff
23 changed files with 2417 additions and 60 deletions
+14 -3
View File
@@ -43,6 +43,7 @@ def dequantize_nvfp4_to_dtype(
device,
block_size=16,
is_sf_128x4_layout=True,
is_sf_linear_layout=False,
):
"""Dequantize the fp4 tensor back to high precision."""
# Two fp4 values are packed into one uint8.
@@ -52,7 +53,9 @@ def dequantize_nvfp4_to_dtype(
tensor_f32 = break_fp4_bytes(tensor_fp4, dtype)
tensor_f32 = tensor_f32.reshape(m, k // block_size, block_size)
tensor_sf = tensor_sf.view(torch.float8_e4m3fn)
if is_sf_128x4_layout:
if is_sf_linear_layout:
tensor_sf = tensor_sf[:m, : k // block_size]
elif is_sf_128x4_layout:
tensor_sf = convert_swizzled_to_linear(tensor_sf, m, k, block_size)
else:
tensor_sf = convert_swizzled_8x4_layout_to_linear(tensor_sf, m, k, block_size)
@@ -146,7 +149,15 @@ def get_nvfp4_global_scale(a: torch.Tensor):
return (FLOAT8_E4M3_MAX * FLOAT4_E2M1_MAX) / torch.abs(a).max().to(torch.float32)
def quant_nvfp4_tensor(a: torch.Tensor):
def quant_nvfp4_tensor(
a: torch.Tensor,
*,
is_sf_swizzled_layout: bool = True,
):
a_global_scale = get_nvfp4_global_scale(a)
a_quant, a_block_scale = scaled_fp4_quant(a, a_global_scale)
a_quant, a_block_scale = scaled_fp4_quant(
a,
a_global_scale,
is_sf_swizzled_layout=is_sf_swizzled_layout,
)
return a_quant, a_block_scale, a_global_scale