[Migration] Migrate GGUF quantization support to plugin (#39612)
Signed-off-by: Isotr0py <mozf@mail2.sysu.edu.cn>
This commit is contained in:
@@ -19,7 +19,6 @@ from transformers import GenerationConfig, PretrainedConfig
|
||||
from transformers.configuration_utils import ALLOWED_LAYER_TYPES
|
||||
from transformers.models.auto.image_processing_auto import get_image_processor_config
|
||||
from transformers.models.auto.modeling_auto import (
|
||||
MODEL_FOR_CAUSAL_LM_MAPPING_NAMES,
|
||||
MODEL_MAPPING_NAMES,
|
||||
)
|
||||
from transformers.models.auto.tokenization_auto import get_tokenizer_config
|
||||
@@ -35,12 +34,6 @@ from vllm.transformers_utils.utils import (
|
||||
from vllm.utils.torch_utils import common_broadcastable_dtype
|
||||
|
||||
from .config_parser_base import ConfigParserBase
|
||||
from .gguf_utils import (
|
||||
check_gguf_file,
|
||||
is_gguf,
|
||||
is_remote_gguf,
|
||||
split_remote_gguf,
|
||||
)
|
||||
from .repo_utils import (
|
||||
file_or_path_exists,
|
||||
get_hf_file_to_dict,
|
||||
@@ -611,17 +604,9 @@ def maybe_override_with_speculators(
|
||||
Returns:
|
||||
Tuple of (resolved_model, resolved_tokenizer, speculative_config)
|
||||
"""
|
||||
if check_gguf_file(model):
|
||||
kwargs["gguf_file"] = Path(model).name
|
||||
gguf_model_repo = Path(model).parent
|
||||
elif is_remote_gguf(model):
|
||||
repo_id, _ = split_remote_gguf(model)
|
||||
gguf_model_repo = Path(repo_id)
|
||||
else:
|
||||
gguf_model_repo = None
|
||||
kwargs["local_files_only"] = huggingface_hub.constants.HF_HUB_OFFLINE
|
||||
config_dict, _ = PretrainedConfig.get_config_dict(
|
||||
model if gguf_model_repo is None else gguf_model_repo,
|
||||
model,
|
||||
revision=revision,
|
||||
token=hf_token,
|
||||
**without_trust_remote_code(kwargs),
|
||||
@@ -659,21 +644,6 @@ def get_config(
|
||||
hf_overrides_fn: Callable[[PretrainedConfig], PretrainedConfig] | None = None,
|
||||
**kwargs,
|
||||
) -> PretrainedConfig:
|
||||
# Separate model folder from file path for GGUF models
|
||||
|
||||
_is_gguf = is_gguf(model)
|
||||
_is_remote_gguf = is_remote_gguf(model)
|
||||
if _is_gguf:
|
||||
if check_gguf_file(model):
|
||||
# Local GGUF file
|
||||
kwargs["gguf_file"] = Path(model).name
|
||||
model = Path(model).parent
|
||||
elif _is_remote_gguf:
|
||||
# Remote GGUF - extract repo_id from repo_id:quant_type format
|
||||
# The actual GGUF file will be downloaded later by GGUFModelLoader
|
||||
# Keep model as repo_id:quant_type for download, but use repo_id for config
|
||||
model, _ = split_remote_gguf(model)
|
||||
|
||||
if config_format == "auto":
|
||||
try:
|
||||
# First check for Mistral to avoid defaulting to
|
||||
@@ -684,25 +654,8 @@ def get_config(
|
||||
model=model, config_name=MISTRAL_CONFIG_NAME, revision=revision
|
||||
):
|
||||
config_format = "mistral"
|
||||
elif (_is_gguf and not _is_remote_gguf) or file_or_path_exists(
|
||||
model, HF_CONFIG_NAME, revision=revision
|
||||
):
|
||||
elif file_or_path_exists(model, HF_CONFIG_NAME, revision=revision):
|
||||
config_format = "hf"
|
||||
# Remote GGUF models must have config.json in repo,
|
||||
# otherwise the config can't be parsed correctly.
|
||||
# FIXME(Isotr0py): Support remote GGUF repos without config.json
|
||||
elif _is_remote_gguf and not file_or_path_exists(
|
||||
model, HF_CONFIG_NAME, revision=revision
|
||||
):
|
||||
err_msg = (
|
||||
"Could not find config.json for remote GGUF model repo. "
|
||||
"To load remote GGUF model through `<repo_id>:<quant_type>`, "
|
||||
"ensure your model has config.json (HF format) file. "
|
||||
"Otherwise please specify --hf-config-path <original_repo> "
|
||||
"in engine args to fetch config from unquantized hf model."
|
||||
)
|
||||
logger.error(err_msg)
|
||||
raise ValueError(err_msg)
|
||||
else:
|
||||
raise ValueError(
|
||||
"Could not detect config format for no config file found. "
|
||||
@@ -737,34 +690,6 @@ def get_config(
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
# Patching defaults for GGUF models
|
||||
if _is_gguf:
|
||||
# Some models have different default values between GGUF and HF.
|
||||
def apply_gguf_default(key: str, gguf_default: Any):
|
||||
"""
|
||||
Apply GGUF defaults unless explicitly configured.
|
||||
|
||||
This function reads/writes external `config` and `config_dict`.
|
||||
If the specified `key` is not in `config_dict` (i.e. not explicitly
|
||||
configured and the default HF value is used), it updates the
|
||||
corresponding `config` value to `gguf_default`.
|
||||
"""
|
||||
if key not in config_dict:
|
||||
config.update({key: gguf_default})
|
||||
|
||||
# Apply architecture-specific GGUF defaults.
|
||||
if config.model_type in {"qwen3_moe"}:
|
||||
# Qwen3 MoE: norm_topk_prob is always true.
|
||||
# Note that, this parameter is always false (HF default) on Qwen2 MoE.
|
||||
apply_gguf_default("norm_topk_prob", True)
|
||||
|
||||
# Special architecture mapping check for GGUF models
|
||||
if _is_gguf:
|
||||
if config.model_type not in MODEL_FOR_CAUSAL_LM_MAPPING_NAMES:
|
||||
raise RuntimeError(f"Can't get gguf config for {config.model_type}.")
|
||||
model_type = MODEL_FOR_CAUSAL_LM_MAPPING_NAMES[config.model_type]
|
||||
config.update({"architectures": [model_type]})
|
||||
|
||||
# Architecture mapping for models without explicit architectures field
|
||||
if not config.architectures:
|
||||
if config.model_type not in MODEL_MAPPING_NAMES:
|
||||
@@ -856,9 +781,6 @@ def get_pooling_config(
|
||||
A dictionary containing the pooling type and whether
|
||||
normalization is used, or None if no pooling configuration is found.
|
||||
"""
|
||||
if is_remote_gguf(model):
|
||||
model, _ = split_remote_gguf(model)
|
||||
|
||||
modules_file_name = "modules.json"
|
||||
|
||||
modules_dict = None
|
||||
@@ -1074,11 +996,6 @@ def get_hf_image_processor_config(
|
||||
# ModelScope does not provide an interface for image_processor
|
||||
if envs.VLLM_USE_MODELSCOPE:
|
||||
return dict()
|
||||
# Separate model folder from file path for GGUF models
|
||||
if check_gguf_file(model):
|
||||
model = Path(model).parent
|
||||
elif is_remote_gguf(model):
|
||||
model, _ = split_remote_gguf(model)
|
||||
return get_image_processor_config(
|
||||
model, token=hf_token, revision=revision, **kwargs
|
||||
)
|
||||
@@ -1108,13 +1025,6 @@ def try_get_generation_config(
|
||||
config_format: str | ConfigFormat = "auto",
|
||||
hf_token: bool | str | None = None,
|
||||
) -> GenerationConfig | None:
|
||||
# GGUF files don't have generation_config.json - their config is embedded
|
||||
# in the file header. Skip all filesystem lookups to avoid re-reading the
|
||||
# memory-mapped file, which can hang in multi-process scenarios when the
|
||||
# EngineCore process already has the file mapped.
|
||||
if is_gguf(model):
|
||||
return None
|
||||
|
||||
try:
|
||||
return GenerationConfig.from_pretrained(
|
||||
model,
|
||||
|
||||
@@ -1,336 +0,0 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""GGUF utility functions."""
|
||||
|
||||
from functools import cache
|
||||
from os import PathLike
|
||||
from pathlib import Path
|
||||
|
||||
import gguf
|
||||
import regex as re
|
||||
from gguf.constants import Keys, VisionProjectorType
|
||||
from gguf.quants import GGMLQuantizationType
|
||||
from transformers import Gemma3Config, PretrainedConfig, SiglipVisionConfig
|
||||
|
||||
from vllm.logger import init_logger
|
||||
|
||||
from .repo_utils import list_filtered_repo_files
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
@cache
|
||||
def check_gguf_file(model: str | PathLike) -> bool:
|
||||
"""Check if the file is a GGUF model."""
|
||||
model = Path(model)
|
||||
if not model.is_file():
|
||||
return False
|
||||
elif model.suffix == ".gguf":
|
||||
return True
|
||||
|
||||
try:
|
||||
with model.open("rb") as f:
|
||||
header = f.read(4)
|
||||
|
||||
return header == b"GGUF"
|
||||
except Exception as e:
|
||||
logger.debug("Error reading file %s: %s", model, e)
|
||||
return False
|
||||
|
||||
|
||||
@cache
|
||||
def is_remote_gguf(model: str | Path) -> bool:
|
||||
"""Check if the model is a remote GGUF model.
|
||||
|
||||
Recognizes two forms:
|
||||
1. Standard: ``repo_id:quant_type`` where *quant_type* is a known
|
||||
GGML quantization type (e.g. ``Q4_K_M``).
|
||||
2. Non-standard: ``repo_id:quant_type`` where *quant_type* contains
|
||||
a known GGML type with extra prefixes (e.g. ``UD-Q4_K_XL``).
|
||||
A warning is logged and actual file existence is validated later
|
||||
during download.
|
||||
"""
|
||||
pattern = r"^[a-zA-Z0-9][a-zA-Z0-9._-]*/[a-zA-Z0-9][a-zA-Z0-9._-]*:[A-Za-z0-9_+-]+$"
|
||||
model = str(model)
|
||||
if re.fullmatch(pattern, model):
|
||||
_, quant_type = model.rsplit(":", 1)
|
||||
if is_valid_gguf_quant_type(quant_type):
|
||||
return True
|
||||
if is_nonstandard_gguf_quant_type(quant_type):
|
||||
logger.warning(
|
||||
"Non-standard GGUF quant type '%s' detected.",
|
||||
quant_type,
|
||||
)
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def is_nonstandard_gguf_quant_type(quant_type: str) -> bool:
|
||||
"""Check if a non-standard quant type contains a known GGML type.
|
||||
|
||||
Splits the quant type by the last ``-`` and checks whether the
|
||||
trailing part is a standard GGML type. For example::
|
||||
|
||||
UD-Q4_K_XL → rsplit → ["UD", "Q4_K_XL"] → Q4_K_XL valid ✓
|
||||
UD-IQ4_NL → rsplit → ["UD", "IQ4_NL"] → IQ4_NL valid ✓
|
||||
Custom-UD-Q4_K → rsplit → ["Custom-UD", "Q4_K"] → Q4_K valid ✓
|
||||
RANDOM → no "-" → False
|
||||
"""
|
||||
if "-" not in quant_type:
|
||||
return False
|
||||
_, remainder = quant_type.rsplit("-", 1)
|
||||
return is_valid_gguf_quant_type(remainder)
|
||||
|
||||
|
||||
# Common suffixes used in GGUF file naming conventions
|
||||
# e.g., Q4_K_M, Q3_K_S, Q5_K_L, Q2_K_XL
|
||||
_GGUF_QUANT_SUFFIXES = ("_M", "_S", "_L", "_XL", "_XS", "_XXS")
|
||||
|
||||
|
||||
def is_valid_gguf_quant_type(gguf_quant_type: str) -> bool:
|
||||
"""Check if the quant type is a valid GGUF quant type.
|
||||
|
||||
Supports both exact GGML quant types (e.g., Q4_K, IQ1_S) and
|
||||
extended naming conventions (e.g., Q4_K_M, Q3_K_S, Q5_K_L).
|
||||
"""
|
||||
# Check for exact match first
|
||||
if getattr(GGMLQuantizationType, gguf_quant_type, None) is not None:
|
||||
return True
|
||||
|
||||
# Check for extended naming conventions (e.g., Q4_K_M -> Q4_K)
|
||||
for suffix in _GGUF_QUANT_SUFFIXES:
|
||||
if gguf_quant_type.endswith(suffix):
|
||||
base_type = gguf_quant_type[: -len(suffix)]
|
||||
if getattr(GGMLQuantizationType, base_type, None) is not None:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
|
||||
def split_remote_gguf(model: str | Path) -> tuple[str, str]:
|
||||
"""Split the model into repo_id and quant type."""
|
||||
model = str(model)
|
||||
if is_remote_gguf(model):
|
||||
parts = model.rsplit(":", 1)
|
||||
return (parts[0], parts[1])
|
||||
raise ValueError(
|
||||
f"Wrong GGUF model or invalid GGUF quant type: {model}.\n"
|
||||
"- It should be in repo_id:quant_type format.\n"
|
||||
f"- Valid base quant types: {GGMLQuantizationType._member_names_}\n"
|
||||
f"- Extended suffixes also supported: {_GGUF_QUANT_SUFFIXES}\n"
|
||||
"- Non-standard GGUF quant types also supported: "
|
||||
"dash-separated prefixes (e.g. UD-Q4_K_XL, Custom-Q8_0)",
|
||||
)
|
||||
|
||||
|
||||
def is_gguf(model: str | Path) -> bool:
|
||||
"""Check if the model is a GGUF model.
|
||||
|
||||
Args:
|
||||
model: Model name, path, or Path object to check.
|
||||
|
||||
Returns:
|
||||
True if the model is a GGUF model, False otherwise.
|
||||
"""
|
||||
model = str(model)
|
||||
|
||||
# Check if it's a local GGUF file
|
||||
if check_gguf_file(model):
|
||||
return True
|
||||
|
||||
# Check if it's a remote GGUF model (repo_id:quant_type format)
|
||||
return is_remote_gguf(model)
|
||||
|
||||
|
||||
def detect_gguf_multimodal(model: str) -> Path | None:
|
||||
"""Check if GGUF model has multimodal projector file.
|
||||
|
||||
Args:
|
||||
model: Model path string
|
||||
|
||||
Returns:
|
||||
Path to mmproj file if found, None otherwise
|
||||
"""
|
||||
if not model.endswith(".gguf"):
|
||||
return None
|
||||
|
||||
try:
|
||||
model_path = Path(model)
|
||||
if not model_path.is_file():
|
||||
return None
|
||||
|
||||
model_dir = model_path.parent
|
||||
mmproj_patterns = ["mmproj.gguf", "mmproj-*.gguf", "*mmproj*.gguf"]
|
||||
for pattern in mmproj_patterns:
|
||||
mmproj_files = list(model_dir.glob(pattern))
|
||||
if mmproj_files:
|
||||
return mmproj_files[0]
|
||||
return None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
def extract_vision_config_from_gguf(mmproj_path: str) -> "SiglipVisionConfig | None":
|
||||
"""Extract vision config parameters from mmproj.gguf metadata.
|
||||
|
||||
Reads vision encoder configuration from GGUF metadata fields using
|
||||
standardized GGUF constants. Automatically detects the projector type
|
||||
(e.g., gemma3, llama4) and applies model-specific parameters accordingly.
|
||||
|
||||
The function extracts standard CLIP vision parameters from GGUF metadata
|
||||
and applies projector-type-specific customizations. For unknown projector
|
||||
types, it uses safe defaults from SiglipVisionConfig.
|
||||
|
||||
Args:
|
||||
mmproj_path: Path to mmproj.gguf file (str or Path)
|
||||
|
||||
Returns:
|
||||
SiglipVisionConfig if extraction succeeds, None if any required
|
||||
field is missing from the GGUF metadata
|
||||
|
||||
Raises:
|
||||
Exception: Exceptions from GGUF reading (file not found, corrupted
|
||||
file, etc.) propagate directly from gguf.GGUFReader
|
||||
"""
|
||||
reader = gguf.GGUFReader(str(mmproj_path))
|
||||
|
||||
# Detect projector type to apply model-specific parameters
|
||||
projector_type = None
|
||||
projector_type_field = reader.get_field(Keys.Clip.PROJECTOR_TYPE)
|
||||
if projector_type_field:
|
||||
try:
|
||||
projector_type = bytes(projector_type_field.parts[-1]).decode("utf-8")
|
||||
except (AttributeError, UnicodeDecodeError) as e:
|
||||
logger.warning("Failed to decode projector type from GGUF: %s", e)
|
||||
|
||||
# Map GGUF field constants to SiglipVisionConfig parameters.
|
||||
# Uses official GGUF constants from gguf-py for standardization.
|
||||
# Format: {gguf_constant: (param_name, dtype)}
|
||||
VISION_CONFIG_FIELDS = {
|
||||
Keys.ClipVision.EMBEDDING_LENGTH: ("hidden_size", int),
|
||||
Keys.ClipVision.FEED_FORWARD_LENGTH: ("intermediate_size", int),
|
||||
Keys.ClipVision.BLOCK_COUNT: ("num_hidden_layers", int),
|
||||
Keys.ClipVision.Attention.HEAD_COUNT: ("num_attention_heads", int),
|
||||
Keys.ClipVision.IMAGE_SIZE: ("image_size", int),
|
||||
Keys.ClipVision.PATCH_SIZE: ("patch_size", int),
|
||||
Keys.ClipVision.Attention.LAYERNORM_EPS: ("layer_norm_eps", float),
|
||||
}
|
||||
|
||||
# Extract and validate all required fields
|
||||
config_params = {}
|
||||
for gguf_key, (param_name, dtype) in VISION_CONFIG_FIELDS.items():
|
||||
field = reader.get_field(gguf_key)
|
||||
if field is None:
|
||||
logger.warning(
|
||||
"Missing required vision config field '%s' in mmproj.gguf",
|
||||
gguf_key,
|
||||
)
|
||||
return None
|
||||
# Extract scalar value from GGUF field and convert to target type
|
||||
config_params[param_name] = dtype(field.parts[-1])
|
||||
|
||||
# Apply model-specific parameters based on projector type
|
||||
if projector_type == VisionProjectorType.GEMMA3:
|
||||
# Gemma3 doesn't use the vision pooling head (multihead attention)
|
||||
# This is a vLLM-specific parameter used in SiglipVisionTransformer
|
||||
config_params["vision_use_head"] = False
|
||||
logger.info("Detected Gemma3 projector, disabling vision pooling head")
|
||||
# Add other projector-type-specific customizations here as needed
|
||||
# elif projector_type == VisionProjectorType.LLAMA4:
|
||||
# config_params["vision_use_head"] = ...
|
||||
|
||||
# Create config with extracted parameters
|
||||
# Note: num_channels and attention_dropout use SiglipVisionConfig defaults
|
||||
# (3 and 0.0 respectively) which are correct for all models
|
||||
config = SiglipVisionConfig(**config_params)
|
||||
|
||||
if projector_type:
|
||||
logger.info(
|
||||
"Extracted vision config from mmproj.gguf (projector_type: %s)",
|
||||
projector_type,
|
||||
)
|
||||
else:
|
||||
logger.info("Extracted vision config from mmproj.gguf metadata")
|
||||
|
||||
return config
|
||||
|
||||
|
||||
def maybe_patch_hf_config_from_gguf(
|
||||
model: str,
|
||||
hf_config: PretrainedConfig,
|
||||
) -> PretrainedConfig:
|
||||
"""Patch HF config for GGUF models.
|
||||
|
||||
Applies GGUF-specific patches to HuggingFace config:
|
||||
1. For multimodal models: patches architecture and vision config
|
||||
2. For all GGUF models: overrides vocab_size from embedding tensor
|
||||
|
||||
This ensures compatibility with GGUF models that have extended
|
||||
vocabularies (e.g., Unsloth) where the GGUF file contains more
|
||||
tokens than the HuggingFace tokenizer config specifies.
|
||||
|
||||
Args:
|
||||
model: Model path string
|
||||
hf_config: HuggingFace config to patch in-place
|
||||
|
||||
Returns:
|
||||
Updated HuggingFace config
|
||||
"""
|
||||
# Patch multimodal config if mmproj.gguf exists
|
||||
mmproj_path = detect_gguf_multimodal(model)
|
||||
if mmproj_path is not None:
|
||||
vision_config = extract_vision_config_from_gguf(str(mmproj_path))
|
||||
|
||||
# Create HF config for Gemma3 multimodal
|
||||
text_config = hf_config.get_text_config()
|
||||
is_gemma3 = hf_config.model_type in ("gemma3", "gemma3_text")
|
||||
if vision_config is not None and is_gemma3:
|
||||
new_hf_config = Gemma3Config(
|
||||
text_config=text_config,
|
||||
vision_config=vision_config,
|
||||
architectures=["Gemma3ForConditionalGeneration"],
|
||||
)
|
||||
hf_config = new_hf_config
|
||||
|
||||
return hf_config
|
||||
|
||||
|
||||
def get_gguf_file_path_from_hf(
|
||||
repo_id: str | Path,
|
||||
quant_type: str,
|
||||
revision: str | None = None,
|
||||
) -> str:
|
||||
"""Get the GGUF file path from HuggingFace Hub based on repo_id and quant_type.
|
||||
|
||||
Args:
|
||||
repo_id: The HuggingFace repository ID (e.g., "Qwen/Qwen3-0.6B")
|
||||
quant_type: The quantization type (e.g., "Q4_K_M", "F16")
|
||||
revision: Optional revision/branch name
|
||||
|
||||
Returns:
|
||||
The path to the GGUF file on HuggingFace Hub (e.g., "filename.gguf"),
|
||||
"""
|
||||
repo_id = str(repo_id)
|
||||
gguf_patterns = [
|
||||
f"*-{quant_type}.gguf",
|
||||
f"*-{quant_type}-*.gguf",
|
||||
f"*/*-{quant_type}.gguf",
|
||||
f"*/*-{quant_type}-*.gguf",
|
||||
]
|
||||
matching_files = list_filtered_repo_files(
|
||||
repo_id,
|
||||
allow_patterns=gguf_patterns,
|
||||
revision=revision,
|
||||
)
|
||||
|
||||
if len(matching_files) == 0:
|
||||
raise ValueError(
|
||||
"Could not find GGUF file for repo %s with quantization %s.",
|
||||
repo_id,
|
||||
quant_type,
|
||||
)
|
||||
|
||||
# Sort to ensure consistent ordering (prefer non-sharded files)
|
||||
matching_files.sort(key=lambda x: (x.count("-"), x))
|
||||
gguf_filename = matching_files[0]
|
||||
return gguf_filename
|
||||
@@ -25,7 +25,6 @@ from typing_extensions import TypeVar
|
||||
|
||||
from vllm.logger import init_logger
|
||||
from vllm.transformers_utils import processors
|
||||
from vllm.transformers_utils.gguf_utils import is_gguf
|
||||
from vllm.transformers_utils.repo_utils import get_hf_file_to_dict
|
||||
from vllm.transformers_utils.utils import convert_model_repo_to_path
|
||||
from vllm.utils.func_utils import get_allowed_kwarg_only_overrides
|
||||
@@ -181,17 +180,8 @@ _cached_get_video_processor_cls_name = lru_cache(
|
||||
def get_video_processor_cls_name(
|
||||
model_config: "ModelConfig",
|
||||
) -> str | None:
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load video processor metadata."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
return _cached_get_video_processor_cls_name(model, revision=revision)
|
||||
|
||||
|
||||
@@ -375,20 +365,9 @@ def cached_processor_from_config(
|
||||
processor_cls: type[_P] | tuple[type[_P], ...] = ProcessorMixin,
|
||||
**kwargs: Any,
|
||||
) -> _P:
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load processor."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
|
||||
return cached_get_processor_without_dynamic_kwargs(
|
||||
model,
|
||||
revision=revision,
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
processor_cls=processor_cls, # type: ignore[arg-type]
|
||||
**_merge_mm_kwargs(model_config, processor_cls, **kwargs),
|
||||
@@ -489,19 +468,9 @@ def cached_image_processor_from_config(
|
||||
model_config: "ModelConfig",
|
||||
**kwargs: Any,
|
||||
):
|
||||
if is_gguf(model_config.model):
|
||||
assert not is_gguf(model_config.tokenizer), (
|
||||
"For multimodal GGUF models, the original tokenizer "
|
||||
"should be used to correctly load image processor."
|
||||
)
|
||||
model = model_config.tokenizer
|
||||
revision = model_config.tokenizer_revision
|
||||
else:
|
||||
model = model_config.model
|
||||
revision = model_config.revision
|
||||
return cached_get_image_processor(
|
||||
model,
|
||||
revision=revision,
|
||||
model_config.model,
|
||||
revision=model_config.revision,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
**_merge_mm_kwargs(model_config, AutoImageProcessor, **kwargs),
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user