[ Kernel ] FP8 Dynamic-Per-Token Quant Kernel (#6511)
Co-authored-by: Varun Sundar Rabindranath <varun@neuralmagic.com>
This commit is contained in:
co-authored by
Varun Sundar Rabindranath
parent
e76466dde2
commit
b5241e41d9
@@ -335,6 +335,17 @@ def scaled_fp8_quant(
|
||||
return output, scale
|
||||
|
||||
|
||||
def dynamic_per_token_scaled_fp8_quant(
|
||||
input: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:
|
||||
|
||||
output = torch.empty_like(input, dtype=torch.float8_e4m3fn)
|
||||
scales = torch.empty((input.numel() // input.shape[-1], 1),
|
||||
device=input.device,
|
||||
dtype=torch.float32)
|
||||
torch.ops._C.dynamic_per_token_scaled_fp8_quant(output, input, scales)
|
||||
return output, scales
|
||||
|
||||
|
||||
# int8
|
||||
def scaled_int8_quant(
|
||||
input: torch.Tensor,
|
||||
|
||||
Reference in New Issue
Block a user