[Model][VLM] Add Qwen2.5-Omni model support (thinker only) (#15130)
Signed-off-by: fyabc <suyang.fy@alibaba-inc.com> Signed-off-by: Roger Wang <ywang@roblox.com> Co-authored-by: Roger Wang <136131678+ywang96@users.noreply.github.com> Co-authored-by: Roger Wang <ywang@roblox.com> Co-authored-by: Xiong Wang <wangxiongts@163.com>
This commit is contained in:
co-authored by
Roger Wang
Roger Wang
Xiong Wang
parent
5c9121203c
commit
2c1bd848a6
@@ -222,8 +222,7 @@ def patch_rope_scaling_dict(rope_scaling: Dict[str, Any]) -> None:
|
||||
logger.warning("Replacing legacy rope_type 'mrope' with 'default'")
|
||||
|
||||
|
||||
def uses_mrope(config: PretrainedConfig) -> bool:
|
||||
"""Detect if the model with this config uses M-ROPE."""
|
||||
def _uses_mrope(config: PretrainedConfig) -> bool:
|
||||
rope_scaling = getattr(config, "rope_scaling", None)
|
||||
if rope_scaling is None:
|
||||
return False
|
||||
@@ -231,6 +230,24 @@ def uses_mrope(config: PretrainedConfig) -> bool:
|
||||
return "mrope_section" in rope_scaling
|
||||
|
||||
|
||||
def uses_mrope(config: PretrainedConfig) -> bool:
|
||||
"""Detect if the model with this config uses M-ROPE."""
|
||||
return _uses_mrope(config) or thinker_uses_mrope(config)
|
||||
|
||||
|
||||
def thinker_uses_mrope(config: PretrainedConfig) -> bool:
|
||||
"""Detect if the model contains a thinker config and it uses M-ROPE."""
|
||||
thinker_config = getattr(config, "thinker_config", None)
|
||||
if thinker_config is None:
|
||||
return False
|
||||
|
||||
thinker_text_config = getattr(thinker_config, "text_config", None)
|
||||
if thinker_text_config is None:
|
||||
return False
|
||||
|
||||
return uses_mrope(thinker_text_config)
|
||||
|
||||
|
||||
def is_encoder_decoder(config: PretrainedConfig) -> bool:
|
||||
"""Detect if the model with this config is used as an encoder/decoder."""
|
||||
text_config = getattr(config, "text_config", None)
|
||||
@@ -740,6 +757,11 @@ def get_hf_text_config(config: PretrainedConfig):
|
||||
# if transformers config doesn't align with this assumption.
|
||||
assert hasattr(config.text_config, "num_attention_heads")
|
||||
return config.text_config
|
||||
elif hasattr(config, "thinker_config"):
|
||||
# TODO(suyang.fy): Refactor code.
|
||||
# For Qwen2.5-Omni, change hf_text_config to
|
||||
# thinker_config.text_config.
|
||||
return config.thinker_config.text_config
|
||||
else:
|
||||
return config
|
||||
|
||||
|
||||
@@ -111,6 +111,55 @@ def cached_processor_from_config(
|
||||
)
|
||||
|
||||
|
||||
def get_feature_extractor(
|
||||
processor_name: str,
|
||||
*args: Any,
|
||||
trust_remote_code: bool = False,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""Load an audio feature extractor for the given model name
|
||||
via HuggingFace."""
|
||||
# don't put this import at the top level
|
||||
# it will call torch.cuda.device_count()
|
||||
from transformers import AutoFeatureExtractor
|
||||
from transformers.feature_extraction_utils import FeatureExtractionMixin
|
||||
try:
|
||||
feature_extractor = AutoFeatureExtractor.from_pretrained(
|
||||
processor_name,
|
||||
*args,
|
||||
trust_remote_code=trust_remote_code,
|
||||
**kwargs)
|
||||
except ValueError as e:
|
||||
# If the error pertains to the processor class not existing or not
|
||||
# currently being imported, suggest using the --trust-remote-code flag.
|
||||
# Unlike AutoTokenizer, AutoImageProcessor does not separate such errors
|
||||
if not trust_remote_code:
|
||||
err_msg = (
|
||||
"Failed to load the feature extractor. If the feature "
|
||||
"extractor is a custom extractor not yet available in the "
|
||||
"HuggingFace transformers library, consider setting "
|
||||
"`trust_remote_code=True` in LLM or using the "
|
||||
"`--trust-remote-code` flag in the CLI.")
|
||||
raise RuntimeError(err_msg) from e
|
||||
else:
|
||||
raise e
|
||||
return cast(FeatureExtractionMixin, feature_extractor)
|
||||
|
||||
|
||||
cached_get_feature_extractor = lru_cache(get_feature_extractor)
|
||||
|
||||
|
||||
def cached_feature_extractor_from_config(
|
||||
model_config: "ModelConfig",
|
||||
**kwargs: Any,
|
||||
):
|
||||
return cached_get_feature_extractor(
|
||||
model_config.model,
|
||||
trust_remote_code=model_config.trust_remote_code,
|
||||
**_merge_mm_kwargs(model_config, **kwargs),
|
||||
)
|
||||
|
||||
|
||||
def get_image_processor(
|
||||
processor_name: str,
|
||||
*args: Any,
|
||||
|
||||
Reference in New Issue
Block a user