iHore/vLLM
8
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

fix: NVFP4 quantization out_dtype should match model dtype, not torch default (#48861)

Browse Source 
Signed-off-by: Chris Fontes <2224082+fattchris@users.noreply.github.com>
Co-authored-by: Chris Fontes <chris@fontes.io>
Co-authored-by: Chris Fontes <chris@fontes.dev>
Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
This commit is contained in:
Chris Fontes
2026-08-03 22:47:37 +00:00
committed by GitHub
co-authored by Chris Fontes Chris Fontes mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
parent f57123aa2d
commit 0b37d8389f
1 changed files with 3 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vllm/model_executor/layers/quantization/modelopt.py
+3 -3
View File
@@ -450,7 +450,7 @@ class ModelOptFp8LinearMethod(LinearMethodBase):
def __init__(self, quant_config: ModelOptFp8Config) -> None:
self.quant_config = quant_config
self.out_dtype = torch.get_default_dtype()
self.out_dtype = get_current_vllm_config().model_config.dtype
self.input_dtype = get_current_vllm_config().model_config.dtype
def create_weights(
@@ -543,7 +543,7 @@ class ModelOptFp8PcPtLinearMethod(LinearMethodBase):
def __init__(self, quant_config: ModelOptFp8Config) -> None:
self.quant_config = quant_config
self.out_dtype = torch.get_default_dtype()
self.out_dtype = get_current_vllm_config().model_config.dtype
self.input_dtype = get_current_vllm_config().model_config.dtype
def create_weights(
@@ -640,7 +640,7 @@ class ModelOptFp8PbWoLinearMethod(LinearMethodBase):
static=True, group_shape=GroupShape(block_n, block_k)
)
self.out_dtype = torch.get_default_dtype()
self.out_dtype = get_current_vllm_config().model_config.dtype
self.input_dtype = get_current_vllm_config().model_config.dtype
def create_weights(