"refactor: refactor_repeated_interfaces" (#32486)
Signed-off-by: tom-zju <tanjianpingzju1990@gmail.com> Co-authored-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com>
This commit is contained in:
@@ -40,6 +40,7 @@ from .siglip import SiglipVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_layer_index,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -300,21 +301,15 @@ def _get_num_hidden_layers(hf_config: AyaVisionConfig) -> int:
|
||||
num_hidden_layers = hf_config.vision_config.num_hidden_layers
|
||||
# If we have one feature layer, initialize up to that layer
|
||||
if isinstance(feature_layers, int):
|
||||
return _get_layer_index(feature_layers, num_hidden_layers)
|
||||
return get_layer_index(feature_layers, num_hidden_layers)
|
||||
# If we have multiple feature layers, initialize up to the deepest m
|
||||
elif isinstance(feature_layers, (list, tuple)):
|
||||
return max(_get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
raise TypeError(
|
||||
f"vision_layer_feature type: {type(feature_layers)} is not supported"
|
||||
)
|
||||
|
||||
|
||||
def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
|
||||
if feature_layer_index < 0:
|
||||
return num_hidden_layers + feature_layer_index + 1
|
||||
return feature_layer_index
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
AyaVisionMultiModalProcessor,
|
||||
info=AyaVisionProcessingInfo,
|
||||
|
||||
@@ -63,6 +63,7 @@ from .siglip import SiglipVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_layer_index,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -442,29 +443,15 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
|
||||
num_hidden_layers = hf_config.vision_config.num_hidden_layers
|
||||
# If we have one feature layer, initialize up to that layer
|
||||
if isinstance(feature_layers, int):
|
||||
return _get_layer_index(feature_layers, num_hidden_layers)
|
||||
return get_layer_index(feature_layers, num_hidden_layers)
|
||||
# If we have multiple feature layers, initialize up to the deepest one
|
||||
elif isinstance(feature_layers, (list, tuple)):
|
||||
return max(_get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
raise TypeError(
|
||||
f"vision_layer_feature type: {type(feature_layers)} is not supported"
|
||||
)
|
||||
|
||||
|
||||
def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
|
||||
"""Given a signed vision feature layer, get the number of hidden layers
|
||||
needed to leverage it.
|
||||
|
||||
Args:
|
||||
feature_layer_index: Index of a required layer in the visual encoder.
|
||||
num_hidden_layers: The total number of hidden layers in the visual
|
||||
encoder.
|
||||
"""
|
||||
if feature_layer_index < 0:
|
||||
return num_hidden_layers + feature_layer_index + 1
|
||||
return feature_layer_index
|
||||
|
||||
|
||||
def init_vision_tower_for_llava(
|
||||
hf_config: LlavaLikeConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
|
||||
@@ -52,6 +52,7 @@ from .pixtral import PixtralHFEncoderInfo, PixtralHFVisionModel
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
WeightsMapper,
|
||||
get_layer_index,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -369,29 +370,15 @@ def _get_num_hidden_layers(hf_config: LlavaLikeConfig) -> int:
|
||||
num_hidden_layers = hf_config.vision_config.num_hidden_layers
|
||||
# If we have one feature layer, initialize up to that layer
|
||||
if isinstance(feature_layers, int):
|
||||
return _get_layer_index(feature_layers, num_hidden_layers)
|
||||
return get_layer_index(feature_layers, num_hidden_layers)
|
||||
# If we have multiple feature layers, initialize up to the deepest one
|
||||
elif isinstance(feature_layers, (list, tuple)):
|
||||
return max(_get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
return max(get_layer_index(idx, num_hidden_layers) for idx in feature_layers)
|
||||
raise TypeError(
|
||||
f"vision_layer_feature type: {type(feature_layers)} is not supported"
|
||||
)
|
||||
|
||||
|
||||
def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
|
||||
"""Given a signed vision feature layer, get the number of hidden layers
|
||||
needed to leverage it.
|
||||
|
||||
Args:
|
||||
feature_layer_index: Index of a required layer in the visual encoder.
|
||||
num_hidden_layers: The total number of hidden layers in the visual
|
||||
encoder.
|
||||
"""
|
||||
if feature_layer_index < 0:
|
||||
return num_hidden_layers + feature_layer_index + 1
|
||||
return feature_layer_index
|
||||
|
||||
|
||||
def init_vision_tower_for_llava(
|
||||
hf_config: LlavaLikeConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
|
||||
@@ -47,7 +47,12 @@ from vllm.utils.tensor_schema import TensorSchema, TensorShape
|
||||
from .clip import CLIPVisionModel
|
||||
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
|
||||
from .siglip import SiglipVisionModel
|
||||
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
|
||||
from .utils import (
|
||||
AutoWeightsLoader,
|
||||
get_layer_index,
|
||||
init_vllm_registered_model,
|
||||
maybe_prefix,
|
||||
)
|
||||
from .vision import (
|
||||
VisionEncoderInfo,
|
||||
get_num_selected_vision_tokens,
|
||||
@@ -356,18 +361,13 @@ def init_vision_tower_for_tarsier(
|
||||
feature_layers = hf_config.vision_feature_layer
|
||||
base_num_hidden_layers = vision_config.num_hidden_layers
|
||||
|
||||
def _get_layer_index(feature_layer_index: int, num_hidden_layers_total: int) -> int:
|
||||
if feature_layer_index < 0:
|
||||
return num_hidden_layers_total + feature_layer_index + 1
|
||||
return feature_layer_index
|
||||
|
||||
if isinstance(feature_layers, int):
|
||||
num_hidden_layers_to_init = _get_layer_index(
|
||||
num_hidden_layers_to_init = get_layer_index(
|
||||
feature_layers, base_num_hidden_layers
|
||||
)
|
||||
elif isinstance(feature_layers, (list, tuple)):
|
||||
num_hidden_layers_to_init = max(
|
||||
_get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers
|
||||
get_layer_index(idx, base_num_hidden_layers) for idx in feature_layers
|
||||
)
|
||||
else:
|
||||
raise TypeError(
|
||||
|
||||
@@ -830,3 +830,16 @@ def process_eagle_weight(
|
||||
model.has_own_lm_head = True
|
||||
if "embed_tokens" in name:
|
||||
model.has_own_embed_tokens = True
|
||||
|
||||
|
||||
def get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
|
||||
"""Given a signed vision feature layer, get the number of hidden layers
|
||||
needed to leverage it.
|
||||
|
||||
Args:
|
||||
feature_layer_index: Index of a required layer in the visual encoder.
|
||||
num_hidden_layers: The total number of hidden layers in the visual encoder.
|
||||
"""
|
||||
if feature_layer_index < 0:
|
||||
return num_hidden_layers + feature_layer_index + 1
|
||||
return feature_layer_index
|
||||
|
||||
Reference in New Issue
Block a user