[Bugfix][Quant] Read safetensors headers of checkpoints with other file names (#60411)
Signed-off-by: Stefan Koncarevic <Stefan.Koncarevic@amd.com> Co-authored-by: Andreas Karatzas <akaratza@amd.com>
This commit is contained in:
co-authored by
Andreas Karatzas
parent
ee406adf0c
commit
814301b268
@@ -23,7 +23,10 @@ from vllm.model_executor.layers.quantization.auto_gptq import (
|
||||
AutoGPTQLinearMethod,
|
||||
AutoGPTQMoEMethod,
|
||||
)
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import override_config
|
||||
from vllm.model_executor.layers.quantization.utils.gptq_utils import (
|
||||
is_layer_gptq_quantized,
|
||||
override_config,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
PROMPT = "On the surface of Mars, we found"
|
||||
@@ -71,6 +74,19 @@ def test_auto_gptq_config_get_name():
|
||||
assert AutoGPTQConfig.get_name() == "auto_gptq"
|
||||
|
||||
|
||||
def test_auto_gptq_quantizes_every_layer_when_the_module_list_is_unknown():
|
||||
"""An empty module list means the safetensors metadata could not be read,
|
||||
not that the checkpoint holds no quantized layer."""
|
||||
assert is_layer_gptq_quantized(
|
||||
prefix="model.layers.0.mlp.down_proj",
|
||||
quantized_layers=[],
|
||||
)
|
||||
assert not is_layer_gptq_quantized(
|
||||
prefix="model.layers.0.mlp.down_proj",
|
||||
quantized_layers=["self_attn.q_proj"],
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"dynamic",
|
||||
[
|
||||
|
||||
@@ -241,6 +241,7 @@ def test_safetensors_metadata_of_repo_without_safetensors():
|
||||
)
|
||||
api = SimpleNamespace(
|
||||
get_safetensors_metadata=get_safetensors_metadata,
|
||||
list_repo_files=MagicMock(return_value=["pytorch_model.bin"]),
|
||||
snapshot_download=MagicMock(side_effect=LocalEntryNotFoundError("no cache")),
|
||||
)
|
||||
|
||||
@@ -250,6 +251,34 @@ def test_safetensors_metadata_of_repo_without_safetensors():
|
||||
get_safetensors_metadata.assert_called_once()
|
||||
|
||||
|
||||
def test_safetensors_metadata_of_repo_with_a_nonstandard_file_name():
|
||||
"""`get_safetensors_metadata` only knows `model.safetensors` and its index,
|
||||
so older checkpoints are read through the file listing instead."""
|
||||
from huggingface_hub.errors import NotASafetensorsRepoError
|
||||
from huggingface_hub.utils import TensorInfo
|
||||
|
||||
weights = "gptq_model-4bit-128g.safetensors"
|
||||
tensor = TensorInfo(dtype="I32", shape=[5632], data_offsets=(0, 22528))
|
||||
parse_safetensors_file_metadata = MagicMock(
|
||||
return_value=SimpleNamespace(tensors={"layers.0.mlp.down_proj.qweight": tensor})
|
||||
)
|
||||
api = SimpleNamespace(
|
||||
get_safetensors_metadata=MagicMock(
|
||||
side_effect=NotASafetensorsRepoError("not a safetensors repo")
|
||||
),
|
||||
list_repo_files=MagicMock(return_value=["config.json", weights]),
|
||||
parse_safetensors_file_metadata=parse_safetensors_file_metadata,
|
||||
)
|
||||
|
||||
with patch.object(config_module, "hf_api", lambda: api):
|
||||
metadata = get_safetensors_params_metadata("some/old-gptq-model")
|
||||
|
||||
assert metadata["layers.0.mlp.down_proj.qweight"]["dtype"] == "I32"
|
||||
parse_safetensors_file_metadata.assert_called_once_with(
|
||||
"some/old-gptq-model", weights, revision=None
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("section_key", "mrope_section", "expected_num_dims"),
|
||||
[
|
||||
|
||||
@@ -133,6 +133,11 @@ def is_layer_gptq_quantized(
|
||||
|
||||
quantized_layers = flatten_list(quantized_layers)
|
||||
|
||||
# Empty means no metadata could be read at all, not that the checkpoint
|
||||
# holds no quantized layer, and loading GPTQ weights unquantized cannot work.
|
||||
if not quantized_layers:
|
||||
return True
|
||||
|
||||
# Fused layers like gate_up_proj or qkv_proj will not be fused
|
||||
# in the safetensors checkpoint. So, we convert the name
|
||||
# from the fused version to unfused + check to make sure that
|
||||
|
||||
@@ -1355,6 +1355,30 @@ def _read_safetensors_metadata_in_dir(local_dir: Path) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _read_safetensors_metadata_in_repo(
|
||||
model: str,
|
||||
*,
|
||||
revision: str | None = None,
|
||||
) -> dict[str, Any]:
|
||||
api = hf_api()
|
||||
try:
|
||||
filenames = [
|
||||
filename
|
||||
for filename in api.list_repo_files(model, revision=revision)
|
||||
if filename.endswith(".safetensors")
|
||||
]
|
||||
return {
|
||||
param_name: asdict(info)
|
||||
for filename in filenames
|
||||
for param_name, info in api.parse_safetensors_file_metadata(
|
||||
model, filename, revision=revision
|
||||
).tensors.items()
|
||||
}
|
||||
except Exception as e:
|
||||
logger.debug("Could not read safetensors headers for %s: %s", model, e)
|
||||
return {}
|
||||
|
||||
|
||||
def get_safetensors_params_metadata(
|
||||
model: str,
|
||||
*,
|
||||
@@ -1372,6 +1396,12 @@ def get_safetensors_params_metadata(
|
||||
for param_name, info in file_mt.tensors.items()
|
||||
}
|
||||
|
||||
# `get_safetensors_metadata` only looks for `model.safetensors` and its
|
||||
# index, so older checkpoints naming their weights differently land here
|
||||
# even though their headers are perfectly readable.
|
||||
if metadata := _read_safetensors_metadata_in_repo(model, revision=revision):
|
||||
return metadata
|
||||
|
||||
# Hub fetch failed (e.g. 429, network unreachable). Fall back to the
|
||||
# local HF cache: weights may already be cached from a prior run, and
|
||||
# weight loading itself uses the same cache.
|
||||
@@ -1390,7 +1420,15 @@ def get_safetensors_params_metadata(
|
||||
str(e),
|
||||
)
|
||||
return {}
|
||||
return _read_safetensors_metadata_in_dir(Path(local_dir))
|
||||
|
||||
metadata = _read_safetensors_metadata_in_dir(Path(local_dir))
|
||||
if not metadata:
|
||||
logger.warning_once(
|
||||
"Could not retrieve safetensors metadata for %s "
|
||||
"(Hub fetch failed and the cached snapshot holds no weights yet).",
|
||||
model,
|
||||
)
|
||||
return metadata
|
||||
|
||||
|
||||
@cache
|
||||
|
||||
Reference in New Issue
Block a user