# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project from transformers import PreTrainedConfig class Mimo_VLVisionConfig(PreTrainedConfig): model_type = "mimovl" base_config_key = "vision_config" def __init__( self, depth=28, hidden_size=1280, hidden_act="silu", intermediate_size=4608, num_heads=32, in_channels=3, patch_size=16, spatial_merge_size=2, temporal_patch_size=2, tokens_per_second=2, window_size=128, out_hidden_size=2048, fullatt_block_indexes=None, initializer_range=0.02, kv_channels=64, # HACK qk_channels=64, num_query_groups=4, num_key_value_heads=8, vit_window_attn_types=None, visual_token_window_size=64, **kwargs, ): super().__init__(**kwargs) self.depth = depth self.hidden_size = hidden_size self.hidden_act = hidden_act self.intermediate_size = intermediate_size self.num_heads = num_heads # Support GQA: if num_key_value_heads is not provided, # default to num_heads (MHA) if num_key_value_heads is None: num_key_value_heads = num_heads self.num_key_value_heads = num_key_value_heads self.in_channels = in_channels self.patch_size = patch_size self.spatial_merge_size = spatial_merge_size self.temporal_patch_size = temporal_patch_size self.tokens_per_second = tokens_per_second self.window_size = window_size self.fullatt_block_indexes = ( fullatt_block_indexes if fullatt_block_indexes is not None else [7, 15, 23, 31] ) self.out_hidden_size = out_hidden_size self.initializer_range = initializer_range self.kv_channels = kv_channels self.qk_channels = qk_channels self.num_query_groups = num_query_groups self.vit_window_attn_types = vit_window_attn_types or [-1] * depth self.visual_token_window_size = visual_token_window_size