[Bugfix][Quant] Read safetensors headers of checkpoints with other file names (#60411)

Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com>
Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
stefankoncarevic
2026-10-08 01:38:41 -05:00
committed by GitHub
co-authored by Andreas Karatzas
parent ee406adf0c
commit 814301b268
4 changed files with 90 additions and 2 deletions
+17 -1
View File
@@ -23,7 +23,10 @@ from vllm.model_executor.layers.quantization.auto_gptq import (
AutoGPTQLinearMethod,
AutoGPTQMoEMethod,
)
from vllm.model_executor.layers.quantization.utils.gptq_utils import override_config
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
is_layer_gptq_quantized,
override_config,
)
from vllm.platforms import current_platform
PROMPT = "On the surface of Mars, we found"
@@ -71,6 +74,19 @@ def test_auto_gptq_config_get_name():
assert AutoGPTQConfig.get_name() == "auto_gptq"
def test_auto_gptq_quantizes_every_layer_when_the_module_list_is_unknown():
"""An empty module list means the safetensors metadata could not be read,
not that the checkpoint holds no quantized layer."""
assert is_layer_gptq_quantized(
prefix="model.layers.0.mlp.down_proj",
quantized_layers=[],
)
assert not is_layer_gptq_quantized(
prefix="model.layers.0.mlp.down_proj",
quantized_layers=["self_attn.q_proj"],
)
@pytest.mark.parametrize(
"dynamic",
[
+29
View File
@@ -241,6 +241,7 @@ def test_safetensors_metadata_of_repo_without_safetensors():
)
api = SimpleNamespace(
get_safetensors_metadata=get_safetensors_metadata,
list_repo_files=MagicMock(return_value=["pytorch_model.bin"]),
snapshot_download=MagicMock(side_effect=LocalEntryNotFoundError("no cache")),
)
@@ -250,6 +251,34 @@ def test_safetensors_metadata_of_repo_without_safetensors():
get_safetensors_metadata.assert_called_once()
def test_safetensors_metadata_of_repo_with_a_nonstandard_file_name():
"""`get_safetensors_metadata` only knows `model.safetensors` and its index,
so older checkpoints are read through the file listing instead."""
from huggingface_hub.errors import NotASafetensorsRepoError
from huggingface_hub.utils import TensorInfo
weights = "gptq_model-4bit-128g.safetensors"
tensor = TensorInfo(dtype="I32", shape=[5632], data_offsets=(0, 22528))
parse_safetensors_file_metadata = MagicMock(
return_value=SimpleNamespace(tensors={"layers.0.mlp.down_proj.qweight": tensor})
)
api = SimpleNamespace(
get_safetensors_metadata=MagicMock(
side_effect=NotASafetensorsRepoError("not a safetensors repo")
),
list_repo_files=MagicMock(return_value=["config.json", weights]),
parse_safetensors_file_metadata=parse_safetensors_file_metadata,
)
with patch.object(config_module, "hf_api", lambda: api):
metadata = get_safetensors_params_metadata("some/old-gptq-model")
assert metadata["layers.0.mlp.down_proj.qweight"]["dtype"] == "I32"
parse_safetensors_file_metadata.assert_called_once_with(
"some/old-gptq-model", weights, revision=None
)
@pytest.mark.parametrize(
("section_key", "mrope_section", "expected_num_dims"),
[