From 169988a3c0e0912fc20be2d104a4b76a51ad9fa4 Mon Sep 17 00:00:00 2001 From: Andrew Barnes Date: Wed, 29 Apr 2026 16:46:01 -0400 Subject: [PATCH] [ROCm] Use quant_dtype in per_token_quant instead of hardcoded FP8 (#39121) Signed-off-by: Bortlesboat --- vllm/_aiter_ops.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vllm/_aiter_ops.py b/vllm/_aiter_ops.py index 0250fbfac7..2681fd4a73 100644 --- a/vllm/_aiter_ops.py +++ b/vllm/_aiter_ops.py @@ -778,7 +778,7 @@ def _rocm_aiter_per_token_quant_impl( assert quant_dtype in [torch.int8, FP8_DTYPE] out_shape = x.shape - out = torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device) + out = torch.empty(x.shape, dtype=quant_dtype, device=x.device) if scale is None: scale = torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device) dynamic_per_token_scaled_quant( @@ -798,7 +798,7 @@ def _rocm_aiter_per_token_quant_fake( ) -> tuple[torch.Tensor, torch.Tensor]: out_shape = x.shape return ( - torch.empty(x.shape, dtype=FP8_DTYPE, device=x.device), + torch.empty(x.shape, dtype=quant_dtype, device=x.device), torch.empty((*out_shape[:-1], 1), dtype=torch.float32, device=x.device), )