From 0b37d8389f4b8378adab0d3dfa1beffbb152e303 Mon Sep 17 00:00:00 2001 From: Chris Fontes <2224082+fattchris@users.noreply.github.com> Date: Mon, 3 Aug 2026 16:47:37 -0600 Subject: [PATCH] fix: NVFP4 quantization out_dtype should match model dtype, not torch default (#48861) Signed-off-by: Chris Fontes <2224082+fattchris@users.noreply.github.com> Co-authored-by: Chris Fontes Co-authored-by: Chris Fontes Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com> --- vllm/model_executor/layers/quantization/modelopt.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/vllm/model_executor/layers/quantization/modelopt.py b/vllm/model_executor/layers/quantization/modelopt.py index de0b0655ef..bac19536f4 100644 --- a/vllm/model_executor/layers/quantization/modelopt.py +++ b/vllm/model_executor/layers/quantization/modelopt.py @@ -450,7 +450,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase): def __init__(self, quant_config: ModelOptFp8Config) -> None: self.quant_config = quant_config - self.out_dtype = torch.get_default_dtype() + self.out_dtype = get_current_vllm_config().model_config.dtype self.input_dtype = get_current_vllm_config().model_config.dtype def create_weights( @@ -543,7 +543,7 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase): def __init__(self, quant_config: ModelOptFp8Config) -> None: self.quant_config = quant_config - self.out_dtype = torch.get_default_dtype() + self.out_dtype = get_current_vllm_config().model_config.dtype self.input_dtype = get_current_vllm_config().model_config.dtype def create_weights( @@ -640,7 +640,7 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase): static=True, group_shape=GroupShape(block_n, block_k) ) - self.out_dtype = torch.get_default_dtype() + self.out_dtype = get_current_vllm_config().model_config.dtype self.input_dtype = get_current_vllm_config().model_config.dtype def create_weights(