[Model] Add HyperCLOVAX-SEED-Think-32B vision-language model support (#31471)

Signed-off-by: effortprogrammer <yhjhoward7@gmail.com>
Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
This commit is contained in:
Hojin Yang
2026-03-10 10:59:19 +08:00
committed by GitHub
co-authored by Cyrus Leung
parent 4e95ec111c
commit 0836be3b03
8 changed files with 760 additions and 10 deletions
@@ -325,7 +325,7 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]:
return dict(
fields = dict(
pixel_values_images=MultiModalFieldConfig.batched("image"),
image_sizes_images=MultiModalFieldConfig.batched("image"),
vision_query_lengths_images=MultiModalFieldConfig.batched("image"),
@@ -333,6 +333,8 @@ class HCXVisionMultiModalProcessor(BaseMultiModalProcessor[HCXVisionProcessingIn
vision_query_lengths_videos=MultiModalFieldConfig.batched("video"),
)
return fields
def _build_hcxvision_hf_info(
ctx: InputProcessingContext,
@@ -590,12 +592,26 @@ class HCXVisionCAbstractor(nn.Module):
dummy_inputs=HCXVisionDummyInputsBuilder,
)
class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
"""
HyperCLOVAX-SEED Vision-Language Model (V1 architecture).
Supports:
- HyperCLOVAX-SEED-Vision-Instruct-3B
Uses CLIP/SigLIP as the vision encoder with C-Abstractor projector.
"""
packed_modules_mapping = {
"qkv_proj": ["q_proj", "k_proj", "v_proj"],
"gate_up_proj": ["gate_proj", "up_proj"],
}
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
def __init__(
self,
*,
vllm_config: VllmConfig,
prefix: str = "",
) -> None:
super().__init__()
# init configs
@@ -647,8 +663,9 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self.vision_config = vision_config
self.text_config = text_config
# use_sum_loss = bool(kwargs.pop("use_sum_loss", False))
# self.reduction = self._init_reduction_type(use_sum_loss)
self.make_empty_intermediate_tensors = (
self.language_model.make_empty_intermediate_tensors
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> str | None: