diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 0250fbfac7..2681fd4a73 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -778,7 +778,7 @@ def _rocm_aiter_per_token_quant_impl( assert quant_dtype in [torch.int8, FP8_DTYPE] out_shape = x.shape - out = torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device) + out = torch.empty(x.shape, dtype=quant_dtype, device=x.device) if scale is None: scale = torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device) dynamic_per_token_scaled_quant( @@ -798,7 +798,7 @@ def _rocm_aiter_per_token_quant_fake( ) -> tuple[torch.Tensor, torch.Tensor]: out_shape = x.shape return ( - torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device), + torch.empty(x.shape, dtype=quant_dtype, device=x.device), torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device), )