[Misc] Standardize RoPE handling for Qwen2-VL (#9250)

This commit is contained in:
Cyrus Leung
2024-10-16 13:56:17 +08:00
committed by GitHub
parent ed920135c8
commit 7e7eae338d
16 changed files with 102 additions and 200 deletions
+1 -1
View File
@@ -242,7 +242,7 @@ class DeepseekV2Attention(nn.Module):
bias=False,
quant_config=quant_config,
prefix=f"{prefix}.o_proj")
rope_scaling['type'] = 'deepseek_yarn'
rope_scaling["rope_type"] = 'deepseek_yarn'
self.rotary_emb = get_rope(qk_rope_head_dim,
rotary_dim=qk_rope_head_dim,
max_position=max_position_embeddings,
+1 -1
View File
@@ -179,7 +179,7 @@ class Phi3SmallSelfAttention(nn.Module):
rope_scaling["factor"] = self.rope_position_scale
else:
rope_scaling = {
"type": "linear",
"rope_type": "linear",
"factor": self.rope_position_scale,
}
+4 -4
View File
@@ -34,6 +34,8 @@ from PIL import Image
from transformers.image_utils import (get_image_size,
infer_channel_dimension_format,
to_numpy_array)
from transformers.models.qwen2_vl.configuration_qwen2_vl import (
Qwen2VLConfig, Qwen2VLVisionConfig)
from transformers.models.qwen2_vl.image_processing_qwen2_vl import (
make_batched_images, make_batched_videos, smart_resize)
@@ -62,8 +64,7 @@ from vllm.multimodal.base import MultiModalData
from vllm.multimodal.image import cached_get_image_processor
from vllm.platforms import current_platform
from vllm.sequence import IntermediateTensors, SequenceData
from vllm.transformers_utils.configs.qwen2vl import (Qwen2VLConfig,
Qwen2VLVisionConfig)
from vllm.transformers_utils.config import uses_mrope
from vllm.transformers_utils.processor import get_processor
from vllm.utils import is_cpu
@@ -1061,8 +1062,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
if image_input is None and video_input is None:
inputs_embeds = None
else:
rope_scaling = getattr(self.config, "rope_scaling", {})
if rope_scaling.get("type", None) == "mrope":
if uses_mrope(self.config):
assert positions.ndim == 2 and positions.size(0) == 3, (
"multimodal section rotary embedding requires "
f"(3, seq_len) positions, but got {positions.size()}")