[Quantization] Remove FP4 emulation; Fall-back to marlin for device < 100 (#19563)

This commit is contained in:
Dipika Sikka
2025-06-16 17:33:51 -04:00
committed by GitHub
parent 90f9c2eb5c
commit 6bc7b57315
5 changed files with 79 additions and 60 deletions
@@ -102,3 +102,32 @@ def ref_nvfp4_quant(x, global_scale, block_size):
clipped_x = torch.clamp(scaled_x, -6.0, 6.0).reshape(m, n)
# both outputs are float32
return cast_to_fp4(clipped_x), scale.squeeze(-1)
def run_nvfp4_emulations(x: torch.Tensor, input_global_scale: torch.Tensor,
weight: torch.Tensor,
weight_scale_swizzled: torch.Tensor,
weight_global_scale: torch.Tensor):
group_size = 16
x_m, x_k = x.shape
output_dtype = x.dtype
# quantize input to (FP4 and interleaved block scale)
x_fp4, x_blockscale = ref_nvfp4_quant(x, input_global_scale, group_size)
# dequantize input
x_fp4 = x_fp4.reshape(x_m, x_k // group_size, group_size)
x_blockscale = x_blockscale.unsqueeze(-1) / input_global_scale
x_dq = (x_fp4 * x_blockscale).reshape(x_m, x_k).to(output_dtype)
del x_fp4, x_blockscale
# dequantize weight
w_fp4 = weight.data.view(torch.uint8)
w_dq = dequantize_to_dtype(w_fp4, weight_scale_swizzled.data,
weight_global_scale, output_dtype, x.device,
group_size)
# matmul
out = torch.matmul(x_dq, w_dq.t())
del w_dq, x_dq
return out