diff --git a/vllm/config/load.py b/vllm/config/load.py index f1066c2b9a..b21eab144a 100644 --- a/vllm/config/load.py +++ b/vllm/config/load.py @@ -14,8 +14,10 @@ DEFAULT_SAFETENSORS_PREFETCH_BLOCK_SIZE = 16 * 1024 * 1024 SafetensorsLoadStrategy: TypeAlias = Literal["lazy", "eager", "prefetch", "torchao"] if TYPE_CHECKING: + from vllm.model_executor.model_loader import LoadFormats from vllm.model_executor.model_loader.tensorizer import TensorizerConfig else: + LoadFormats = str TensorizerConfig = Any logger = init_logger(__name__) @@ -25,7 +27,7 @@ logger = init_logger(__name__) class LoadConfig: """Configuration for loading the model weights.""" - load_format: str = "auto" + load_format: str | LoadFormats = "auto" """ The format of the model weights to load. diff --git a/vllm/config/model.py b/vllm/config/model.py index 37549e188e..245af557df 100644 --- a/vllm/config/model.py +++ b/vllm/config/model.py @@ -69,7 +69,7 @@ else: me_models = LazyLoader("model_executor", globals(), "vllm.model_executor.models") LoadConfig = Any ParallelConfig = Any - QuantizationMethods = Any + QuantizationMethods = str LogitsProcessor = Any logger = init_logger(__name__) diff --git a/vllm/engine/arg_utils.py b/vllm/engine/arg_utils.py index 5f96a62a87..8cc219264f 100644 --- a/vllm/engine/arg_utils.py +++ b/vllm/engine/arg_utils.py @@ -127,8 +127,8 @@ if TYPE_CHECKING: from vllm.v1.executor import Executor else: Executor = Any - QuantizationMethods = Any - LoadFormats = Any + QuantizationMethods = str + LoadFormats = str UsageContext = Any