Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
66 lines
2.1 KiB
Python
66 lines
2.1 KiB
Python
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
|
|
from transformers import PreTrainedConfig
|
|
|
|
|
|
class Mimo_VLVisionConfig(PreTrainedConfig):
|
|
model_type = "mimovl"
|
|
base_config_key = "vision_config"
|
|
|
|
def __init__(
|
|
self,
|
|
depth=28,
|
|
hidden_size=1280,
|
|
hidden_act="silu",
|
|
intermediate_size=4608,
|
|
num_heads=32,
|
|
in_channels=3,
|
|
patch_size=16,
|
|
spatial_merge_size=2,
|
|
temporal_patch_size=2,
|
|
tokens_per_second=2,
|
|
window_size=128,
|
|
out_hidden_size=2048,
|
|
fullatt_block_indexes=None,
|
|
initializer_range=0.02,
|
|
kv_channels=64, # HACK
|
|
qk_channels=64,
|
|
num_query_groups=4,
|
|
num_key_value_heads=8,
|
|
vit_window_attn_types=None,
|
|
visual_token_window_size=64,
|
|
**kwargs,
|
|
):
|
|
super().__init__(**kwargs)
|
|
|
|
self.depth = depth
|
|
self.hidden_size = hidden_size
|
|
self.hidden_act = hidden_act
|
|
self.intermediate_size = intermediate_size
|
|
self.num_heads = num_heads
|
|
# Support GQA: if num_key_value_heads is not provided,
|
|
# default to num_heads (MHA)
|
|
if num_key_value_heads is None:
|
|
num_key_value_heads = num_heads
|
|
self.num_key_value_heads = num_key_value_heads
|
|
self.in_channels = in_channels
|
|
self.patch_size = patch_size
|
|
self.spatial_merge_size = spatial_merge_size
|
|
self.temporal_patch_size = temporal_patch_size
|
|
self.tokens_per_second = tokens_per_second
|
|
self.window_size = window_size
|
|
self.fullatt_block_indexes = (
|
|
fullatt_block_indexes
|
|
if fullatt_block_indexes is not None
|
|
else [7, 15, 23, 31]
|
|
)
|
|
self.out_hidden_size = out_hidden_size
|
|
self.initializer_range = initializer_range
|
|
self.kv_channels = kv_channels
|
|
self.qk_channels = qk_channels
|
|
self.num_query_groups = num_query_groups
|
|
self.vit_window_attn_types = vit_window_attn_types or [-1] * depth
|
|
self.visual_token_window_size = visual_token_window_size
|