[Model] Enable tower and connector LoRA for Keye (#51780)
Signed-off-by: shujialiu <liushujia0122@163.com>
This commit is contained in:
@@ -1466,6 +1466,14 @@ class BaseKeyeModule(nn.Module, SupportsMultiModal):
|
|||||||
tower_model="visual.",
|
tower_model="visual.",
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
|
||||||
|
merge_size = self.config.vision_config.spatial_merge_size
|
||||||
|
return num_image_tokens * merge_size**2
|
||||||
|
|
||||||
|
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
|
||||||
|
merge_size = self.config.vision_config.spatial_merge_size
|
||||||
|
return num_vision_tokens // merge_size**2
|
||||||
|
|
||||||
|
|
||||||
@MULTIMODAL_REGISTRY.register_processor(
|
@MULTIMODAL_REGISTRY.register_processor(
|
||||||
KeyeMultiModalProcessor,
|
KeyeMultiModalProcessor,
|
||||||
|
|||||||
Reference in New Issue
Block a user