[Model] Add HyperCLOVAX-SEED-Think-32B vision-language model support (#31471)
Signed-off-by: effortprogrammer <yhjhoward7@gmail.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
co-authored by
Cyrus Leung
parent
4e95ec111c
commit
0836be3b03
@@ -325,7 +325,7 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
|
||||
hf_inputs: BatchFeature,
|
||||
hf_processor_mm_kwargs: Mapping[str, object],
|
||||
) -> Mapping[str, MultiModalFieldConfig]:
|
||||
return dict(
|
||||
fields = dict(
|
||||
pixel_values_images=MultiModalFieldConfig.batched("image"),
|
||||
image_sizes_images=MultiModalFieldConfig.batched("image"),
|
||||
vision_query_lengths_images=MultiModalFieldConfig.batched("image"),
|
||||
@@ -333,6 +333,8 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
|
||||
vision_query_lengths_videos=MultiModalFieldConfig.batched("video"),
|
||||
)
|
||||
|
||||
return fields
|
||||
|
||||
|
||||
def _build_hcxvision_hf_info(
|
||||
ctx: InputProcessingContext,
|
||||
@@ -590,12 +592,26 @@ class HCXVisionCAbstractor(nn.Module):
|
||||
dummy_inputs=HCXVisionDummyInputsBuilder,
|
||||
)
|
||||
class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
"""
|
||||
HyperCLOVAX-SEED Vision-Language Model (V1 architecture).
|
||||
|
||||
Supports:
|
||||
- HyperCLOVAX-SEED-Vision-Instruct-3B
|
||||
|
||||
Uses CLIP/SigLIP as the vision encoder with C-Abstractor projector.
|
||||
"""
|
||||
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
|
||||
"gate_up_proj": ["gate_proj", "up_proj"],
|
||||
}
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
vllm_config: VllmConfig,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
|
||||
# init configs
|
||||
@@ -647,8 +663,9 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
|
||||
self.vision_config = vision_config
|
||||
self.text_config = text_config
|
||||
|
||||
# use_sum_loss = bool(kwargs.pop("use_sum_loss", False))
|
||||
# self.reduction = self._init_reduction_type(use_sum_loss)
|
||||
self.make_empty_intermediate_tensors = (
|
||||
self.language_model.make_empty_intermediate_tensors
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
|
||||
Reference in New Issue
Block a user