[ROCm] Use quant_dtype in per_token_quant instead of hardcoded FP8 (#39121)
Signed-off-by: Bortlesboat <bortstheboat@gmail.com>
This commit is contained in:
+2
-2
@@ -778,7 +778,7 @@ def _rocm_aiter_per_token_quant_impl(
|
||||
assert quant_dtype in [torch.int8, FP8_DTYPE]
|
||||
|
||||
out_shape = x.shape
|
||||
out = torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device)
|
||||
out = torch.empty(x.shape, dtype=quant_dtype, device=x.device)
|
||||
if scale is None:
|
||||
scale = torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device)
|
||||
dynamic_per_token_scaled_quant(
|
||||
@@ -798,7 +798,7 @@ def _rocm_aiter_per_token_quant_fake(
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
out_shape = x.shape
|
||||
return (
|
||||
torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device),
|
||||
torch.empty(x.shape, dtype=quant_dtype, device=x.device),
|
||||
torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device),
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user