[ROCm] Use quant_dtype in per_token_quant instead of hardcoded FP8 (#39121)

Signed-off-by: Bortlesboat <bortstheboat@gmail.com>
This commit is contained in:
Andrew Barnes
2026-04-29 20:46:01 +00:00
committed by GitHub
parent faab189554
commit 169988a3c0
+2 -2
View File
@@ -778,7 +778,7 @@ def _rocm_aiter_per_token_quant_impl(
assert quant_dtype in [torch.int8, FP8_DTYPE]
out_shape = x.shape
out = torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device)
out = torch.empty(x.shape, dtype=quant_dtype, device=x.device)
if scale is None:
scale = torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device)
dynamic_per_token_scaled_quant(
@@ -798,7 +798,7 @@ def _rocm_aiter_per_token_quant_fake(
) -> tuple[torch.Tensor, torch.Tensor]:
out_shape = x.shape
return (
torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device),
torch.empty(x.shape, dtype=quant_dtype, device=x.device),
torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device),
)