[Core][Bugfix] Accept GGUF model without .gguf extension (#8056)

This commit is contained in:
Isotr0py
2024-09-02 08:43:26 -04:00
committed by GitHub
parent e2b2aa5a0f
commit 4ca65a9763
4 changed files with 23 additions and 5 deletions
+2 -1
View File
@@ -16,6 +16,7 @@ from vllm.config import (CacheConfig, DecodingConfig, DeviceConfig,
from vllm.executor.executor_base import ExecutorBase
from vllm.logger import init_logger
from vllm.model_executor.layers.quantization import QUANTIZATION_METHODS
from vllm.transformers_utils.utils import check_gguf_file
from vllm.utils import FlexibleArgumentParser
if TYPE_CHECKING:
@@ -753,7 +754,7 @@ class EngineArgs:
def create_engine_config(self) -> EngineConfig:
# gguf file needs a specific model loader and doesn't use hf_repo
if self.model.endswith(".gguf"):
if check_gguf_file(self.model):
self.quantization = self.load_format = "gguf"
# bitsandbytes quantization needs a specific model loader