[Transformers v5] Vendor HCXVisionConfig for compatibility (#38447)
Signed-off-by: Fang Han <fhan0520@gmail.com> Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.7
parent
67058ca326
commit
c103c02a1a
@@ -946,13 +946,6 @@ _MULTIMODAL_EXAMPLE_MODELS = {
|
||||
"HCXVisionForCausalLM": _HfExamplesInfo(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B",
|
||||
trust_remote_code=True,
|
||||
max_transformers_version="4.57",
|
||||
transformers_version_reason={
|
||||
"vllm": (
|
||||
"Custom config cannot be loaded with Transformers "
|
||||
"v5 because `text_config` is not always set"
|
||||
)
|
||||
},
|
||||
),
|
||||
"HCXVisionV2ForCausalLM": _HfExamplesInfo(
|
||||
"naver-hyperclovax/HyperCLOVAX-SEED-Think-32B",
|
||||
|
||||
@@ -101,6 +101,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict(
|
||||
fireredlid="FireRedLIDConfig",
|
||||
funaudiochat="FunAudioChatConfig",
|
||||
granite4_vision="Granite4VisionConfig",
|
||||
hyperclovax_vlm="HCXVisionConfig",
|
||||
hunyuan_vl="HunYuanVLConfig",
|
||||
hy_v3="HYV3Config",
|
||||
isaac="IsaacConfig",
|
||||
@@ -217,8 +218,9 @@ class HFConfigParser(ConfigParserBase):
|
||||
)
|
||||
else:
|
||||
if model_type in _CONFIG_REGISTRY:
|
||||
# Register the config class to AutoConfig to ensure it's used in future
|
||||
# calls to `from_pretrained`
|
||||
# Register the config class to AutoConfig to ensure it's used
|
||||
# in future calls to `from_pretrained` (e.g. from
|
||||
# AutoTokenizer or AutoProcessor).
|
||||
config_class = _CONFIG_REGISTRY[model_type]
|
||||
config_class.model_type = model_type
|
||||
AutoConfig.register(model_type, config_class, exist_ok=True)
|
||||
|
||||
@@ -37,6 +37,7 @@ _CLASS_TO_MODULE: dict[str, str] = {
|
||||
"HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl",
|
||||
"HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl",
|
||||
"HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl",
|
||||
"HCXVisionConfig": "vllm.transformers_utils.configs.hyperclovax",
|
||||
"HYV3Config": "vllm.transformers_utils.configs.hy_v3",
|
||||
"HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax",
|
||||
"IsaacConfig": "vllm.transformers_utils.configs.isaac",
|
||||
@@ -104,6 +105,7 @@ __all__ = [
|
||||
"HunYuanVLConfig",
|
||||
"HunYuanVLTextConfig",
|
||||
"HunYuanVLVisionConfig",
|
||||
"HCXVisionConfig",
|
||||
"HYV3Config",
|
||||
"HyperCLOVAXConfig",
|
||||
"IsaacConfig",
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
# limitations under the License.
|
||||
"""HyperCLOVA X model configuration."""
|
||||
|
||||
from transformers import AutoConfig
|
||||
from transformers.configuration_utils import PretrainedConfig
|
||||
|
||||
|
||||
@@ -275,3 +276,75 @@ class HyperCLOVAXConfig(PretrainedConfig):
|
||||
auto_map=auto_map,
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
class HCXVisionConfig(PretrainedConfig):
|
||||
"""Vendored HyperCLOVAX Vision config with transformers v5 fix.
|
||||
|
||||
The original remote code config does not handle empty initialization
|
||||
(text_config=None), which breaks transformers v5's @strict validation.
|
||||
|
||||
TODO: Remove this class once HyperCLOVAX is upstreamed to transformers.
|
||||
Tracking PR: https://github.com/huggingface/transformers/pull/44956
|
||||
"""
|
||||
|
||||
model_type = "hyperclovax_vlm"
|
||||
keys_to_ignore_at_inference = ["past_key_values"]
|
||||
|
||||
text_config_attribute_map = {
|
||||
"n_embd": "hidden_size",
|
||||
"n_positions": "max_position_embeddings",
|
||||
"n_head": "num_attention_heads",
|
||||
"n_layer": "num_hidden_layers",
|
||||
}
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
text_config=None,
|
||||
vision_config=None,
|
||||
use_nth_layer=-2,
|
||||
img_start_id=100009,
|
||||
decoder_max_length=4096,
|
||||
anyres=False,
|
||||
unpad=False,
|
||||
max_num_grids=-1,
|
||||
num_queries_vis_abstractor=-1,
|
||||
ignore_index=-100,
|
||||
proj_pos_emb=True,
|
||||
proj_prenorm=False,
|
||||
use_1x1_grid=False,
|
||||
**kwargs,
|
||||
):
|
||||
for key, val in self.text_config_attribute_map.items():
|
||||
if text_config is not None and key in text_config:
|
||||
text_config[val] = text_config.pop(key)
|
||||
|
||||
self.text_config = None
|
||||
if text_config is not None:
|
||||
_text_config = AutoConfig.for_model(text_config["model_type"])
|
||||
self.text_config = _text_config.from_dict(text_config)
|
||||
self.hidden_size = self.text_config.hidden_size
|
||||
|
||||
self.vision_config = None
|
||||
if vision_config is not None:
|
||||
_vision_config = AutoConfig.for_model(
|
||||
vision_config["model_type"])
|
||||
self.vision_config = _vision_config.from_dict(vision_config)
|
||||
|
||||
self.use_nth_layer = use_nth_layer
|
||||
self.decoder_max_length = decoder_max_length
|
||||
self.anyres = anyres
|
||||
self.unpad = unpad
|
||||
self.max_num_grids = max_num_grids
|
||||
self.num_queries_vis_abstractor = num_queries_vis_abstractor
|
||||
self.img_start_id = img_start_id
|
||||
self.ignore_index = ignore_index
|
||||
self.proj_pos_emb = proj_pos_emb
|
||||
self.proj_prenorm = proj_prenorm
|
||||
self.use_1x1_grid = use_1x1_grid
|
||||
super().__init__(**kwargs)
|
||||
|
||||
def get_text_config(self, decoder=False):
|
||||
if self.text_config is not None:
|
||||
return self.text_config
|
||||
return self
|
||||
|
||||
Reference in New Issue
Block a user