[ Kernel ] Enable Dynamic Per Token fp8 (#6547)
This commit is contained in:
+13
-13
@@ -300,6 +300,7 @@ def scaled_fp8_quant(
|
||||
input: torch.Tensor,
|
||||
scale: Optional[torch.Tensor] = None,
|
||||
batch_dim_padding: Optional[int] = None,
|
||||
use_per_token_if_dynamic: bool = False,
|
||||
) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP8 and return quantized tensor and scale.
|
||||
@@ -315,6 +316,8 @@ def scaled_fp8_quant(
|
||||
scale: Optional scaling factor for the FP8 quantization
|
||||
batch_dim_padding: If specified, pad the first dimension
|
||||
of the output to at least this value.
|
||||
use_per_token_if_dynamic: Whether to do per_tensor or per_token
|
||||
in the dynamic quantization case.
|
||||
|
||||
Returns:
|
||||
Tuple[torch.Tensor, torch.Tensor]: The output tensor in FP8 and
|
||||
@@ -328,24 +331,21 @@ def scaled_fp8_quant(
|
||||
else:
|
||||
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
|
||||
if scale is None:
|
||||
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
|
||||
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
|
||||
if use_per_token_if_dynamic:
|
||||
scale = torch.empty((input.numel() // input.shape[-1], 1),
|
||||
device=input.device,
|
||||
dtype=torch.float32)
|
||||
torch.ops._C.dynamic_per_token_scaled_fp8_quant(
|
||||
output, input, scale)
|
||||
else:
|
||||
scale = torch.zeros(1, device=input.device, dtype=torch.float32)
|
||||
torch.ops._C.dynamic_scaled_fp8_quant(output, input, scale)
|
||||
else:
|
||||
torch.ops._C.static_scaled_fp8_quant(output, input, scale)
|
||||
|
||||
return output, scale
|
||||
|
||||
|
||||
def dynamic_per_token_scaled_fp8_quant(
|
||||
input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
|
||||
scales = torch.empty((input.numel() // input.shape[-1], 1),
|
||||
device=input.device,
|
||||
dtype=torch.float32)
|
||||
torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
|
||||
return output, scales
|
||||
|
||||
|
||||
# int8
|
||||
def scaled_int8_quant(
|
||||
input: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user