diff --git a/tests/models/registry.py b/tests/models/registry.py index 21d3a50ce9..ab5869cd3f 100644 --- a/tests/models/registry.py +++ b/tests/models/registry.py @@ -946,13 +946,6 @@ _MULTIMODAL_EXAMPLE_MODELS = { "HCXVisionForCausalLM": _HfExamplesInfo( "naver-hyperclovax/HyperCLOVAX-SEED-Vision-Instruct-3B", trust_remote_code=True, - max_transformers_version="4.57", - transformers_version_reason={ - "vllm": ( - "Custom config cannot be loaded with Transformers " - "v5 because `text_config` is not always set" - ) - }, ), "HCXVisionV2ForCausalLM": _HfExamplesInfo( "naver-hyperclovax/HyperCLOVAX-SEED-Think-32B", diff --git a/vllm/transformers_utils/config.py b/vllm/transformers_utils/config.py index 2f00178ba6..c95df9c107 100644 --- a/vllm/transformers_utils/config.py +++ b/vllm/transformers_utils/config.py @@ -101,6 +101,7 @@ _CONFIG_REGISTRY: dict[str, type[PretrainedConfig]] = LazyConfigDict( fireredlid="FireRedLIDConfig", funaudiochat="FunAudioChatConfig", granite4_vision="Granite4VisionConfig", + hyperclovax_vlm="HCXVisionConfig", hunyuan_vl="HunYuanVLConfig", hy_v3="HYV3Config", isaac="IsaacConfig", @@ -217,8 +218,9 @@ class HFConfigParser(ConfigParserBase): ) else: if model_type in _CONFIG_REGISTRY: - # Register the config class to AutoConfig to ensure it's used in future - # calls to `from_pretrained` + # Register the config class to AutoConfig to ensure it's used + # in future calls to `from_pretrained` (e.g. from + # AutoTokenizer or AutoProcessor). config_class = _CONFIG_REGISTRY[model_type] config_class.model_type = model_type AutoConfig.register(model_type, config_class, exist_ok=True) diff --git a/vllm/transformers_utils/configs/__init__.py b/vllm/transformers_utils/configs/__init__.py index 44abe32c91..99f099adc7 100644 --- a/vllm/transformers_utils/configs/__init__.py +++ b/vllm/transformers_utils/configs/__init__.py @@ -37,6 +37,7 @@ _CLASS_TO_MODULE: dict[str, str] = { "HunYuanVLConfig": "vllm.transformers_utils.configs.hunyuan_vl", "HunYuanVLTextConfig": "vllm.transformers_utils.configs.hunyuan_vl", "HunYuanVLVisionConfig": "vllm.transformers_utils.configs.hunyuan_vl", + "HCXVisionConfig": "vllm.transformers_utils.configs.hyperclovax", "HYV3Config": "vllm.transformers_utils.configs.hy_v3", "HyperCLOVAXConfig": "vllm.transformers_utils.configs.hyperclovax", "IsaacConfig": "vllm.transformers_utils.configs.isaac", @@ -104,6 +105,7 @@ __all__ = [ "HunYuanVLConfig", "HunYuanVLTextConfig", "HunYuanVLVisionConfig", + "HCXVisionConfig", "HYV3Config", "HyperCLOVAXConfig", "IsaacConfig", diff --git a/vllm/transformers_utils/configs/hyperclovax.py b/vllm/transformers_utils/configs/hyperclovax.py index 9fa823743d..839b8ed50b 100644 --- a/vllm/transformers_utils/configs/hyperclovax.py +++ b/vllm/transformers_utils/configs/hyperclovax.py @@ -17,6 +17,7 @@ # limitations under the License. """HyperCLOVA X model configuration.""" +from transformers import AutoConfig from transformers.configuration_utils import PretrainedConfig @@ -275,3 +276,75 @@ class HyperCLOVAXConfig(PretrainedConfig): auto_map=auto_map, **kwargs, ) + + +class HCXVisionConfig(PretrainedConfig): + """Vendored HyperCLOVAX Vision config with transformers v5 fix. + + The original remote code config does not handle empty initialization + (text_config=None), which breaks transformers v5's @strict validation. + + TODO: Remove this class once HyperCLOVAX is upstreamed to transformers. + Tracking PR: https://github.com/huggingface/transformers/pull/44956 + """ + + model_type = "hyperclovax_vlm" + keys_to_ignore_at_inference = ["past_key_values"] + + text_config_attribute_map = { + "n_embd": "hidden_size", + "n_positions": "max_position_embeddings", + "n_head": "num_attention_heads", + "n_layer": "num_hidden_layers", + } + + def __init__( + self, + text_config=None, + vision_config=None, + use_nth_layer=-2, + img_start_id=100009, + decoder_max_length=4096, + anyres=False, + unpad=False, + max_num_grids=-1, + num_queries_vis_abstractor=-1, + ignore_index=-100, + proj_pos_emb=True, + proj_prenorm=False, + use_1x1_grid=False, + **kwargs, + ): + for key, val in self.text_config_attribute_map.items(): + if text_config is not None and key in text_config: + text_config[val] = text_config.pop(key) + + self.text_config = None + if text_config is not None: + _text_config = AutoConfig.for_model(text_config["model_type"]) + self.text_config = _text_config.from_dict(text_config) + self.hidden_size = self.text_config.hidden_size + + self.vision_config = None + if vision_config is not None: + _vision_config = AutoConfig.for_model( + vision_config["model_type"]) + self.vision_config = _vision_config.from_dict(vision_config) + + self.use_nth_layer = use_nth_layer + self.decoder_max_length = decoder_max_length + self.anyres = anyres + self.unpad = unpad + self.max_num_grids = max_num_grids + self.num_queries_vis_abstractor = num_queries_vis_abstractor + self.img_start_id = img_start_id + self.ignore_index = ignore_index + self.proj_pos_emb = proj_pos_emb + self.proj_prenorm = proj_prenorm + self.use_1x1_grid = use_1x1_grid + super().__init__(**kwargs) + + def get_text_config(self, decoder=False): + if self.text_config is not None: + return self.text_config + return self