[torch.compile][ROCm] Fuse quantization onto attention using a torch.compile pass (#16756)
Signed-off-by: Luka Govedič <lgovedic@redhat.com> Co-authored-by: Sage Moore <sage@neuralmagic.com>
This commit is contained in:
co-authored by
Sage Moore
parent
96846bb360
commit
f98548b9da
+7
-1
@@ -1225,6 +1225,7 @@ def scaled_fp8_quant(
|
||||
num_token_padding: Optional[int] = None,
|
||||
scale_ub: Optional[torch.Tensor] = None,
|
||||
use_per_token_if_dynamic: bool = False,
|
||||
output: Optional[torch.Tensor] = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Quantize input tensor to FP8 and return quantized tensor and scale.
|
||||
@@ -1256,7 +1257,12 @@ def scaled_fp8_quant(
|
||||
out_dtype: torch.dtype = current_platform.fp8_dtype()
|
||||
if num_token_padding:
|
||||
shape = (max(num_token_padding, input.shape[0]), shape[1])
|
||||
output = torch.empty(shape, device=input.device, dtype=out_dtype)
|
||||
if output is None:
|
||||
output = torch.empty(shape, device=input.device, dtype=out_dtype)
|
||||
else:
|
||||
assert num_token_padding is None, \
|
||||
"padding not supported if output passed in"
|
||||
assert output.dtype == out_dtype
|
||||
|
||||
if scale is None:
|
||||
if use_per_token_if_dynamic:
|
||||
|
||||
Reference in New Issue
Block a user