From 814301b26800a9d22a7d967a161aa07da4cbb5f2 Mon Sep 17 00:00:00 2001 From: stefankoncarevic Date: Thu, 8 Oct 2026 08:38:41 +0200 Subject: [PATCH] [Bugfix][Quant] Read safetensors headers of checkpoints with other file names (#60411) Signed-off-by: Stefan Koncarevic Co-authored-by: Andreas Karatzas --- tests/quantization/test_auto_gptq.py | 18 ++++++++- tests/transformers_utils/test_config.py | 29 ++++++++++++++ .../layers/quantization/utils/gptq_utils.py | 5 +++ vllm/transformers_utils/config.py | 40 ++++++++++++++++++- 4 files changed, 90 insertions(+), 2 deletions(-) diff --git a/tests/quantization/test_auto_gptq.py b/tests/quantization/test_auto_gptq.py index 6e7791654c..7e54599c80 100644 --- a/tests/quantization/test_auto_gptq.py +++ b/tests/quantization/test_auto_gptq.py @@ -23,7 +23,10 @@ from vllm.model_executor.layers.quantization.auto_gptq import ( AutoGPTQLinearMethod, AutoGPTQMoEMethod, ) -from vllm.model_executor.layers.quantization.utils.gptq_utils import override_config +from vllm.model_executor.layers.quantization.utils.gptq_utils import ( + is_layer_gptq_quantized, + override_config, +) from vllm.platforms import current_platform PROMPT = "On the surface of Mars, we found" @@ -71,6 +74,19 @@ def test_auto_gptq_config_get_name(): assert AutoGPTQConfig.get_name() == "auto_gptq" +def test_auto_gptq_quantizes_every_layer_when_the_module_list_is_unknown(): + """An empty module list means the safetensors metadata could not be read, + not that the checkpoint holds no quantized layer.""" + assert is_layer_gptq_quantized( + prefix="model.layers.0.mlp.down_proj", + quantized_layers=[], + ) + assert not is_layer_gptq_quantized( + prefix="model.layers.0.mlp.down_proj", + quantized_layers=["self_attn.q_proj"], + ) + + @pytest.mark.parametrize( "dynamic", [ diff --git a/tests/transformers_utils/test_config.py b/tests/transformers_utils/test_config.py index 4fd27ba278..9b13e72560 100644 --- a/tests/transformers_utils/test_config.py +++ b/tests/transformers_utils/test_config.py @@ -241,6 +241,7 @@ def test_safetensors_metadata_of_repo_without_safetensors(): ) api = SimpleNamespace( get_safetensors_metadata=get_safetensors_metadata, + list_repo_files=MagicMock(return_value=["pytorch_model.bin"]), snapshot_download=MagicMock(side_effect=LocalEntryNotFoundError("no cache")), ) @@ -250,6 +251,34 @@ def test_safetensors_metadata_of_repo_without_safetensors(): get_safetensors_metadata.assert_called_once() +def test_safetensors_metadata_of_repo_with_a_nonstandard_file_name(): + """`get_safetensors_metadata` only knows `model.safetensors` and its index, + so older checkpoints are read through the file listing instead.""" + from huggingface_hub.errors import NotASafetensorsRepoError + from huggingface_hub.utils import TensorInfo + + weights = "gptq_model-4bit-128g.safetensors" + tensor = TensorInfo(dtype="I32", shape=[5632], data_offsets=(0, 22528)) + parse_safetensors_file_metadata = MagicMock( + return_value=SimpleNamespace(tensors={"layers.0.mlp.down_proj.qweight": tensor}) + ) + api = SimpleNamespace( + get_safetensors_metadata=MagicMock( + side_effect=NotASafetensorsRepoError("not a safetensors repo") + ), + list_repo_files=MagicMock(return_value=["config.json", weights]), + parse_safetensors_file_metadata=parse_safetensors_file_metadata, + ) + + with patch.object(config_module, "hf_api", lambda: api): + metadata = get_safetensors_params_metadata("some/old-gptq-model") + + assert metadata["layers.0.mlp.down_proj.qweight"]["dtype"] == "I32" + parse_safetensors_file_metadata.assert_called_once_with( + "some/old-gptq-model", weights, revision=None + ) + + @pytest.mark.parametrize( ("section_key", "mrope_section", "expected_num_dims"), [ diff --git a/vllm/model_executor/layers/quantization/utils/gptq_utils.py b/vllm/model_executor/layers/quantization/utils/gptq_utils.py index d134abf8dc..024564e6f3 100644 --- a/vllm/model_executor/layers/quantization/utils/gptq_utils.py +++ b/vllm/model_executor/layers/quantization/utils/gptq_utils.py @@ -133,6 +133,11 @@ def is_layer_gptq_quantized( quantized_layers = flatten_list(quantized_layers) + # Empty means no metadata could be read at all, not that the checkpoint + # holds no quantized layer, and loading GPTQ weights unquantized cannot work. + if not quantized_layers: + return True + # Fused layers like gate_up_proj or qkv_proj will not be fused # in the safetensors checkpoint. So, we convert the name # from the fused version to unfused + check to make sure that diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 6a339297ee..c22f6fc3c2 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -1355,6 +1355,30 @@ def _read_safetensors_metadata_in_dir(local_dir: Path) -> dict[str, Any]: } +def _read_safetensors_metadata_in_repo( + model: str, + *, + revision: str | None = None, +) -> dict[str, Any]: + api = hf_api() + try: + filenames = [ + filename + for filename in api.list_repo_files(model, revision=revision) + if filename.endswith(".safetensors") + ] + return { + param_name: asdict(info) + for filename in filenames + for param_name, info in api.parse_safetensors_file_metadata( + model, filename, revision=revision + ).tensors.items() + } + except Exception as e: + logger.debug("Could not read safetensors headers for %s: %s", model, e) + return {} + + def get_safetensors_params_metadata( model: str, *, @@ -1372,6 +1396,12 @@ def get_safetensors_params_metadata( for param_name, info in file_mt.tensors.items() } + # `get_safetensors_metadata` only looks for `model.safetensors` and its + # index, so older checkpoints naming their weights differently land here + # even though their headers are perfectly readable. + if metadata := _read_safetensors_metadata_in_repo(model, revision=revision): + return metadata + # Hub fetch failed (e.g. 429, network unreachable). Fall back to the # local HF cache: weights may already be cached from a prior run, and # weight loading itself uses the same cache. @@ -1390,7 +1420,15 @@ def get_safetensors_params_metadata( str(e), ) return {} - return _read_safetensors_metadata_in_dir(Path(local_dir)) + + metadata = _read_safetensors_metadata_in_dir(Path(local_dir)) + if not metadata: + logger.warning_once( + "Could not retrieve safetensors metadata for %s " + "(Hub fetch failed and the cached snapshot holds no weights yet).", + model, + ) + return metadata @cache