[Bugfix][Model] Fix EXAONE-4.5 image inputs: ViT received un-normalised pixels (#61104)
Signed-off-by: godmook <cmoh4135@naver.com> Co-authored-by: cjackal <44624812+cjackal@users.noreply.github.com>
This commit is contained in:
@@ -36,6 +36,7 @@ from vllm.distributed import parallel_state
|
||||
from vllm.distributed import utils as dist_utils
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import MMEncoderAttention
|
||||
from vllm.model_executor.layers.fusion.mm_input_norm import build_mm_input_norm
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
from vllm.model_executor.layers.linear import QKVParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
@@ -272,6 +273,7 @@ class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
|
||||
vision_config: Exaone4_5_VisionConfig,
|
||||
norm_eps: float = 1e-6,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
input_norm: nn.Module | None = None,
|
||||
prefix: str = "",
|
||||
use_data_parallel: bool = False,
|
||||
) -> None:
|
||||
@@ -279,6 +281,7 @@ class EXAONE4_5_VisionTransformer(Qwen2_5_VisionTransformer):
|
||||
vision_config=vision_config,
|
||||
norm_eps=norm_eps,
|
||||
quant_config=quant_config,
|
||||
input_norm=input_norm,
|
||||
prefix=prefix,
|
||||
)
|
||||
depth = vision_config.depth
|
||||
@@ -335,7 +338,9 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
|
||||
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.multimodal_config = multimodal_config
|
||||
self.video_pruning_rate = multimodal_config.video_pruning_rate
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
)
|
||||
@@ -345,6 +350,7 @@ class Exaone4_5_ForConditionalGeneration(Qwen2_5_VLForConditionalGeneration):
|
||||
config.vision_config,
|
||||
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
|
||||
quant_config=self.quant_config,
|
||||
input_norm=build_mm_input_norm(self.model_config),
|
||||
prefix=maybe_prefix(prefix, "visual"),
|
||||
use_data_parallel=self.use_data_parallel,
|
||||
)
|
||||
|
||||
@@ -75,6 +75,7 @@ class Exaone4_5MultiTokenPredictor(ExaoneMoeMultiTokenPredictor):
|
||||
self.layers = nn.ModuleList(
|
||||
Exaone4DecoderLayer(
|
||||
text_config,
|
||||
cache_config=vllm_config.cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.layers.{idx}",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user