"refactor: refactor_repeated_interfaces" (#32486)

Signed-off-by: tom-zju <tanjianpingzju1990@gmail.com>
Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
tjp_zju
2026-01-18 22:07:01 +08:00
committed by GitHub
co-authored by Wentao Ye
parent 38bf2ffb21
commit 2f03035a61
11 changed files with 43 additions and 77 deletions
+3 -8
View File
@@ -40,6 +40,7 @@ from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_layer_index,
init_vllm_registered_model,
maybe_prefix,
)
@@ -300,21 +301,15 @@ def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:
num_hidden_layers = hf_config.vision_config.num_hidden_layers
# If we have one feature layer, initialize up to that layer
if isinstance(feature_layers, int):
return _get_layer_index(feature_layers, num_hidden_layers)
return get_layer_index(feature_layers, num_hidden_layers)
# If we have multiple feature layers, initialize up to the deepest m
elif isinstance(feature_layers, (list, tuple)):
return max(_get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
raise TypeError(
f"vision_layer_feature type: {type(feature_layers)} is not supported"
)
def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
if feature_layer_index < 0:
return num_hidden_layers + feature_layer_index + 1
return feature_layer_index
@MULTIMODAL_REGISTRY.register_processor(
AyaVisionMultiModalProcessor,
info=AyaVisionProcessingInfo,
+3 -16
View File
@@ -63,6 +63,7 @@ from .siglip import SiglipVisionModel
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_layer_index,
init_vllm_registered_model,
maybe_prefix,
)
@@ -442,29 +443,15 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
num_hidden_layers = hf_config.vision_config.num_hidden_layers
# If we have one feature layer, initialize up to that layer
if isinstance(feature_layers, int):
return _get_layer_index(feature_layers, num_hidden_layers)
return get_layer_index(feature_layers, num_hidden_layers)
# If we have multiple feature layers, initialize up to the deepest one
elif isinstance(feature_layers, (list, tuple)):
return max(_get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
raise TypeError(
f"vision_layer_feature type: {type(feature_layers)} is not supported"
)
def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
"""Given a signed vision feature layer, get the number of hidden layers
needed to leverage it.
Args:
feature_layer_index: Index of a required layer in the visual encoder.
num_hidden_layers: The total number of hidden layers in the visual
encoder.
"""
if feature_layer_index < 0:
return num_hidden_layers + feature_layer_index + 1
return feature_layer_index
def init_vision_tower_for_llava(
hf_config: LlavaLikeConfig,
quant_config: QuantizationConfig | None,
+3 -16
View File
@@ -52,6 +52,7 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
from .utils import (
AutoWeightsLoader,
WeightsMapper,
get_layer_index,
init_vllm_registered_model,
maybe_prefix,
)
@@ -369,29 +370,15 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
num_hidden_layers = hf_config.vision_config.num_hidden_layers
# If we have one feature layer, initialize up to that layer
if isinstance(feature_layers, int):
return _get_layer_index(feature_layers, num_hidden_layers)
return get_layer_index(feature_layers, num_hidden_layers)
# If we have multiple feature layers, initialize up to the deepest one
elif isinstance(feature_layers, (list, tuple)):
return max(_get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
raise TypeError(
f"vision_layer_feature type: {type(feature_layers)} is not supported"
)
def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
"""Given a signed vision feature layer, get the number of hidden layers
needed to leverage it.
Args:
feature_layer_index: Index of a required layer in the visual encoder.
num_hidden_layers: The total number of hidden layers in the visual
encoder.
"""
if feature_layer_index < 0:
return num_hidden_layers + feature_layer_index + 1
return feature_layer_index
def init_vision_tower_for_llava(
hf_config: LlavaLikeConfig,
quant_config: QuantizationConfig | None,
+8 -8
View File
@@ -47,7 +47,12 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
from .clip import CLIPVisionModel
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .siglip import SiglipVisionModel
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
from .utils import (
AutoWeightsLoader,
get_layer_index,
init_vllm_registered_model,
maybe_prefix,
)
from .vision import (
VisionEncoderInfo,
get_num_selected_vision_tokens,
@@ -356,18 +361,13 @@ def init_vision_tower_for_tarsier(
feature_layers = hf_config.vision_feature_layer
base_num_hidden_layers = vision_config.num_hidden_layers
def _get_layer_index(feature_layer_index: int, num_hidden_layers_total: int) -> int:
if feature_layer_index < 0:
return num_hidden_layers_total + feature_layer_index + 1
return feature_layer_index
if isinstance(feature_layers, int):
num_hidden_layers_to_init = _get_layer_index(
num_hidden_layers_to_init = get_layer_index(
feature_layers, base_num_hidden_layers
)
elif isinstance(feature_layers, (list, tuple)):
num_hidden_layers_to_init = max(
_get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers
get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers
)
else:
raise TypeError(
+13
View File
@@ -830,3 +830,16 @@ def process_eagle_weight(
model.has_own_lm_head = True
if "embed_tokens" in name:
model.has_own_embed_tokens = True
def get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
"""Given a signed vision feature layer, get the number of hidden layers
needed to leverage it.
Args:
feature_layer_index: Index of a required layer in the visual encoder.
num_hidden_layers: The total number of hidden layers in the visual encoder.
"""
if feature_layer_index < 0:
return num_hidden_layers + feature_layer_index + 1
return feature_layer_index