[ Kernel ] FP8 Dynamic-Per-Token Quant Kernel (#6511)

Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
Varun Sundar Rabindranath
2024-07-18 01:38:35 +00:00
committed by GitHub
co-authored by Varun Sundar Rabindranath
parent e76466dde2
commit b5241e41d9
7 changed files with 274 additions and 43 deletions
+11
View File
@@ -335,6 +335,17 @@ def scaled_fp8_quant(
return output, scale
def dynamic_per_token_scaled_fp8_quant(
input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
scales = torch.empty((input.numel() // input.shape[-1], 1),
device=input.device,
dtype=torch.float32)
torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
return output, scales
# int8
def scaled_int8_quant(
input: torch.Tensor,