[Mypy] Fix mypy typing for Qwen and Qianfan models (#58046)
Signed-off-by: Ashraf Bhuiyan <mbhuiyan@redhat.com>
This commit is contained in:
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
|
||||
|
||||
EXCLUDE = [
|
||||
r"vllm/model_executor/models/[kK]",
|
||||
r"vllm/model_executor/models/[qQ]",
|
||||
r"vllm/model_executor/models/[uU]",
|
||||
r"vllm/model_executor/models/[vV]",
|
||||
r"vllm/model_executor/models/[wW]",
|
||||
|
||||
@@ -1262,7 +1262,7 @@ class SupportsRealtime(Protocol):
|
||||
Override in subclasses based on the model's expected output length."""
|
||||
|
||||
@classmethod
|
||||
async def buffer_realtime_audio(
|
||||
def buffer_realtime_audio(
|
||||
cls,
|
||||
audio_stream: AsyncGenerator[np.ndarray, None],
|
||||
input_stream: asyncio.Queue[list[int]],
|
||||
|
||||
@@ -36,14 +36,14 @@ class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
|
||||
config = self.get_hf_config()
|
||||
vision_config = config.vision_config
|
||||
|
||||
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
kwargs.setdefault("image_size", vision_config.image_size)
|
||||
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
|
||||
merged_kwargs.setdefault("image_size", vision_config.image_size)
|
||||
merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
|
||||
merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
|
||||
merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
|
||||
merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail)
|
||||
|
||||
image_processor = InternVLImageProcessor(**kwargs)
|
||||
image_processor = InternVLImageProcessor(**merged_kwargs)
|
||||
image_size = image_processor.image_size
|
||||
patch_size = vision_config.patch_size
|
||||
downsample_ratio = config.downsample_ratio
|
||||
@@ -73,7 +73,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
|
||||
"""
|
||||
|
||||
def _patch_quant_config(
|
||||
self, config: PreTrainedConfig, quant_config: QuantizationConfig
|
||||
self, config: PreTrainedConfig, quant_config: QuantizationConfig | None
|
||||
) -> None:
|
||||
super()._patch_quant_config(config, quant_config)
|
||||
# ignore vit layers to preserve model performance
|
||||
|
||||
@@ -187,6 +187,12 @@ class Qwen2Attention(nn.Module):
|
||||
if attn_type == AttentionType.ENCODER_ONLY
|
||||
else Attention
|
||||
)
|
||||
attention_kwargs: dict[str, Any] = {}
|
||||
if dual_chunk_attention_config:
|
||||
attention_kwargs = {
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
self.attn = attn_cls(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -196,12 +202,7 @@ class Qwen2Attention(nn.Module):
|
||||
quant_config=quant_config,
|
||||
attn_type=attn_type,
|
||||
prefix=f"{prefix}.attn",
|
||||
**{
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
if dual_chunk_attention_config
|
||||
else {},
|
||||
**attention_kwargs,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -42,6 +42,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
|
||||
Qwen2_5OmniProcessor,
|
||||
)
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.multimodal import MultiModalDummyOptions
|
||||
@@ -63,7 +64,10 @@ from vllm.model_executor.models.qwen2_audio import (
|
||||
Qwen2AudioProcessingInfo,
|
||||
_get_feat_extract_output_lengths,
|
||||
)
|
||||
from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser
|
||||
from vllm.model_executor.models.qwen2_vl import (
|
||||
Qwen2VLMultiModalDataParser,
|
||||
Qwen2VLProcessingInfo,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import (
|
||||
ImageItem,
|
||||
@@ -79,6 +83,7 @@ from vllm.multimodal.parse import (
|
||||
)
|
||||
from vllm.multimodal.processing import (
|
||||
BaseDummyInputsBuilder,
|
||||
BaseProcessingInfo,
|
||||
)
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseMultiModalProcessor,
|
||||
@@ -118,6 +123,14 @@ except (ImportError, ModuleNotFoundError):
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tensor:
|
||||
data = feature.data
|
||||
assert data is not None
|
||||
tensor = data[key].data
|
||||
assert isinstance(tensor, torch.Tensor)
|
||||
return tensor
|
||||
|
||||
|
||||
def unpad_and_flat_audio_features(
|
||||
input_audio_features: torch.Tensor,
|
||||
audio_feature_lengths: torch.Tensor,
|
||||
@@ -347,7 +360,7 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
|
||||
def _parse_audio_data(
|
||||
self,
|
||||
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
|
||||
) -> ModalityDataItems[Any, Any]:
|
||||
) -> ModalityDataItems[Any, Any] | None:
|
||||
if isinstance(data, dict):
|
||||
required, optional = self.embedding_field_sets("audio")
|
||||
data = dict(data)
|
||||
@@ -406,7 +419,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int] | None = None,
|
||||
) -> Mapping[str, int] | None:
|
||||
) -> Mapping[str, int]:
|
||||
mm_counts = mm_counts or {}
|
||||
requested_modalities = {m for m, c in mm_counts.items() if c > 0}
|
||||
mm_max_tokens: dict[str, int] = {}
|
||||
@@ -436,9 +449,13 @@ class Qwen2_5OmniThinkerProcessingInfo(
|
||||
return mm_max_tokens
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]
|
||||
):
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo)
|
||||
_VI = TypeVar(
|
||||
"_VI", bound=Qwen2VLProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo
|
||||
)
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[_VI]):
|
||||
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
num_images = mm_counts.get("image", 0)
|
||||
@@ -462,7 +479,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: MultiModalDummyOptions,
|
||||
) -> MultiModalDataDict:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
feature_extractor = self.info.get_hf_processor().feature_extractor
|
||||
|
||||
target_audio_length = (
|
||||
min(
|
||||
@@ -501,9 +518,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
|
||||
return mm_data
|
||||
|
||||
|
||||
class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
|
||||
):
|
||||
class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]):
|
||||
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
|
||||
return self.dummy_inputs.get_dummy_text(mm_counts)
|
||||
|
||||
@@ -651,6 +666,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
for item in mm_kwargs["video"]:
|
||||
if item and item.get("use_audio_in_video"):
|
||||
use_audio_in_video_tensor = item["use_audio_in_video"].data
|
||||
assert isinstance(use_audio_in_video_tensor, torch.Tensor)
|
||||
if use_audio_in_video_tensor.numel() > 0:
|
||||
use_audio_in_video = bool(use_audio_in_video_tensor.item())
|
||||
break
|
||||
@@ -749,6 +765,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
) -> Sequence[PromptUpdate]:
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(self.info, Qwen2VLProcessingInfo)
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
@@ -786,6 +803,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
if num_features == 0:
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
audio = audios.get(item_idx)
|
||||
assert audio is not None
|
||||
raise ValueError(
|
||||
f"The audio {audio} (len={len(audio)}) is too short "
|
||||
"to be represented inside the model"
|
||||
@@ -887,6 +905,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
|
||||
|
||||
|
||||
class Qwen2_5OmniConditionalGenerationMixin:
|
||||
audio_tower: nn.Module
|
||||
visual: nn.Module
|
||||
vllm_config: VllmConfig
|
||||
|
||||
def _parse_and_validate_audio_input(
|
||||
self, **kwargs: object
|
||||
) -> Qwen2_5OmniAudioFeatureInputs | None:
|
||||
@@ -916,7 +938,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
|
||||
def _parse_and_validate_image_input(
|
||||
self,
|
||||
**kwargs: dict[str, Any],
|
||||
**kwargs: object,
|
||||
) -> Qwen2_5_VLImageInputs | None:
|
||||
pixel_values = kwargs.pop("pixel_values", None)
|
||||
image_embeds = kwargs.pop("image_embeds", None)
|
||||
@@ -932,7 +954,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
if image_embeds is not None:
|
||||
else:
|
||||
return Qwen2_5_VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds,
|
||||
@@ -941,7 +963,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self,
|
||||
**kwargs: dict[str, Any],
|
||||
**kwargs: object,
|
||||
) -> Qwen2_5_VLVideoInputs | None:
|
||||
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
||||
video_embeds = kwargs.pop("video_embeds", None)
|
||||
@@ -957,7 +979,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
else:
|
||||
if not isinstance(video_embeds, torch.Tensor):
|
||||
raise ValueError(
|
||||
"Incorrect type of video embeddings. "
|
||||
@@ -1130,7 +1152,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[
|
||||
str,
|
||||
Qwen2_5_VLImageInputs
|
||||
| Qwen2_5_VLVideoInputs
|
||||
| Qwen2_5OmniAudioFeatureInputs
|
||||
| None,
|
||||
] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -1175,8 +1203,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
f
|
||||
for f in mm_features
|
||||
if f.modality == "video"
|
||||
and f.data is not None
|
||||
and f.data.get("use_audio_in_video")
|
||||
and f.data["use_audio_in_video"].data.item()
|
||||
and _get_feature_tensor(f, "use_audio_in_video").item()
|
||||
]
|
||||
audios = [f for f in mm_features if f.modality == "audio"]
|
||||
|
||||
@@ -1185,7 +1214,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
paired_audio_offsets: set[int] = set()
|
||||
for i, video_f in enumerate(videos_with_audio):
|
||||
if i < len(audios):
|
||||
audio_len = audios[i].data["audio_feature_lengths"].data.item()
|
||||
audio_len = int(
|
||||
_get_feature_tensor(audios[i], "audio_feature_lengths").item()
|
||||
)
|
||||
mapping[video_f.mm_position.offset] = audio_len
|
||||
paired_audio_offsets.add(audios[i].mm_position.offset)
|
||||
return mapping, paired_audio_offsets
|
||||
@@ -1220,9 +1251,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
for mm_feature in sorted_features:
|
||||
offset = mm_feature.mm_position.offset
|
||||
modality = mm_feature.modality
|
||||
assert mm_feature.data is not None
|
||||
|
||||
if modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
|
||||
yield (
|
||||
offset,
|
||||
"image",
|
||||
@@ -1234,16 +1266,16 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
},
|
||||
)
|
||||
elif modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
|
||||
second_per_grid_ts = 1.0
|
||||
if mm_feature.data.get("second_per_grid_ts"):
|
||||
second_per_grid_ts = mm_feature.data[
|
||||
"second_per_grid_ts"
|
||||
].data.item()
|
||||
second_per_grid_ts = _get_feature_tensor(
|
||||
mm_feature, "second_per_grid_ts"
|
||||
).item()
|
||||
use_audio_in_video = False
|
||||
if mm_feature.data.get("use_audio_in_video"):
|
||||
use_audio_in_video = bool(
|
||||
mm_feature.data["use_audio_in_video"].data.item()
|
||||
_get_feature_tensor(mm_feature, "use_audio_in_video").item()
|
||||
)
|
||||
|
||||
yield (
|
||||
@@ -1261,7 +1293,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
|
||||
elif modality == "audio":
|
||||
# Skip audio that's paired with video (handled in video case)
|
||||
if offset not in paired_audio_offsets:
|
||||
audio_len = mm_feature.data["audio_feature_lengths"].data.item()
|
||||
audio_len = _get_feature_tensor(
|
||||
mm_feature, "audio_feature_lengths"
|
||||
).item()
|
||||
yield offset, "audio", {"audio_feature_length": audio_len}
|
||||
|
||||
def _compute_interleaved_positions(
|
||||
|
||||
@@ -654,7 +654,10 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
self.spatial_merge_size = vision_config.spatial_merge_size
|
||||
self.fullatt_block_indexes = vision_config.fullatt_block_indexes
|
||||
self.spatial_merge_unit = self.spatial_merge_size**2
|
||||
self._rope_by_thw_cache = LRUCache(capacity=1024)
|
||||
self._rope_by_thw_cache: LRUCache[
|
||||
tuple[int, int, int],
|
||||
tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor],
|
||||
] = LRUCache(capacity=1024)
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.tp_size = (
|
||||
1
|
||||
@@ -885,7 +888,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
|
||||
def prepare_encoder_metadata(
|
||||
self,
|
||||
grid_thw: list[list[int]],
|
||||
grid_thw: Sequence[Sequence[int]],
|
||||
*,
|
||||
max_batch_size: int | None = None,
|
||||
max_frames_per_batch: int | None = None,
|
||||
@@ -927,9 +930,9 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
# patchify
|
||||
rotary_pos_emb_cos = []
|
||||
rotary_pos_emb_sin = []
|
||||
window_index: list = []
|
||||
cu_window_seqlens: list = [torch.tensor([0], dtype=torch.int32)]
|
||||
cu_seqlens: list = []
|
||||
window_indices: list[torch.Tensor] = []
|
||||
cu_window_seqlens_parts = [torch.tensor([0], dtype=torch.int32)]
|
||||
cu_seqlens_parts: list[torch.Tensor] = []
|
||||
|
||||
window_index_id = 0
|
||||
cu_window_seqlens_last = 0
|
||||
@@ -946,26 +949,26 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
cu_seqlens_thw,
|
||||
) = self.get_rope_by_thw(t, h, w)
|
||||
|
||||
window_index.append(window_index_thw + window_index_id)
|
||||
window_indices.append(window_index_thw + window_index_id)
|
||||
window_index_id += t * llm_h * llm_w
|
||||
|
||||
cu_seqlens_window_thw = cu_seqlens_window_thw + cu_window_seqlens_last
|
||||
cu_window_seqlens_last = cu_seqlens_window_thw[-1]
|
||||
cu_window_seqlens.append(cu_seqlens_window_thw)
|
||||
cu_window_seqlens_parts.append(cu_seqlens_window_thw)
|
||||
|
||||
rotary_pos_emb_cos.append(cos_thw)
|
||||
rotary_pos_emb_sin.append(sin_thw)
|
||||
|
||||
cu_seqlens.append(cu_seqlens_thw)
|
||||
cu_seqlens_parts.append(cu_seqlens_thw)
|
||||
|
||||
rotary_pos_emb_cos = torch.cat(rotary_pos_emb_cos)
|
||||
rotary_pos_emb_sin = torch.cat(rotary_pos_emb_sin)
|
||||
window_index = torch.cat(window_index)
|
||||
window_index = torch.cat(window_indices)
|
||||
# compute reverse indices
|
||||
reverse_indices = self.invert_permutation(window_index)
|
||||
cu_window_seqlens = torch.cat(cu_window_seqlens)
|
||||
cu_window_seqlens = torch.cat(cu_window_seqlens_parts)
|
||||
cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
|
||||
cu_seqlens = torch.cat(cu_seqlens)
|
||||
cu_seqlens = torch.cat(cu_seqlens_parts)
|
||||
cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)
|
||||
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
|
||||
|
||||
@@ -1088,6 +1091,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
|
||||
|
||||
seq_len = hidden_states.shape[0]
|
||||
if encoder_metadata is None:
|
||||
assert grid_thw is not None
|
||||
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
|
||||
|
||||
rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
|
||||
@@ -1162,7 +1166,7 @@ class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
|
||||
)
|
||||
videos = [v.copy() for v in videos]
|
||||
|
||||
video_items = []
|
||||
video_items: list[VideoItem] = []
|
||||
for video in videos:
|
||||
video_num_frames = video.shape[0]
|
||||
video_metadata = {
|
||||
@@ -1216,6 +1220,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
hf_data = hf_inputs.hf_data
|
||||
|
||||
if videos := hf_data.get("videos"):
|
||||
assert isinstance(videos, list)
|
||||
# HF expects video metadata as a separate argument.
|
||||
hf_data["videos"] = [video for video, _ in videos]
|
||||
hf_data["video_metadata"] = [
|
||||
@@ -1336,17 +1341,22 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
offset = mm_feature.mm_position.offset
|
||||
data = mm_feature.data
|
||||
assert data is not None
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
grid = data["image_grid_thw"].data
|
||||
assert isinstance(grid, torch.Tensor)
|
||||
t, h, w = grid.tolist()
|
||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||
yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
|
||||
elif mm_feature.modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
grid = data["video_grid_thw"].data
|
||||
assert isinstance(grid, torch.Tensor)
|
||||
t, h, w = grid.tolist()
|
||||
second_per_grid_ts = 1.0
|
||||
if mm_feature.data.get("second_per_grid_ts", None):
|
||||
second_per_grid_ts = mm_feature.data[
|
||||
"second_per_grid_ts"
|
||||
].data.item()
|
||||
if seconds := data.get("second_per_grid_ts"):
|
||||
assert isinstance(seconds.data, torch.Tensor)
|
||||
second_per_grid_ts = seconds.data.item()
|
||||
t_factor = second_per_grid_ts * tokens_per_second
|
||||
yield (
|
||||
offset,
|
||||
@@ -1409,7 +1419,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.config = config
|
||||
@@ -1458,7 +1468,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
if image_embeds is not None:
|
||||
else:
|
||||
return Qwen2_5_VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds,
|
||||
@@ -1484,7 +1494,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
second_per_grid_ts=second_per_grid_ts,
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
else:
|
||||
return Qwen2_5_VLVideoEmbeddingInputs(
|
||||
type="video_embeds",
|
||||
video_embeds=video_embeds,
|
||||
@@ -1549,8 +1559,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
)
|
||||
emb = torch.cat([emb, positions], dim=1)
|
||||
image_embeds_out.append(emb)
|
||||
image_embeds_split = image_embeds_out
|
||||
return tuple(image_embeds_split)
|
||||
return tuple(image_embeds_out)
|
||||
|
||||
def _process_video_input(
|
||||
self, video_input: Qwen2_5_VLVideoInputs
|
||||
@@ -1614,6 +1623,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
tokens_per_second = self.config.vision_config.tokens_per_second
|
||||
|
||||
video_embeds_out = []
|
||||
assert self.video_pruning_rate is not None
|
||||
for emb, size, video_second_per_grid_t in zip(
|
||||
video_embeds_split, grid_thw_list, second_per_grid_ts
|
||||
):
|
||||
@@ -1702,7 +1712,9 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
return mm_embeddings_out, positions, mrope_positions_delta
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[
|
||||
str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
|
||||
] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -1808,6 +1820,7 @@ class Qwen2_5_VLForConditionalGeneration(
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
mm_registry = MULTIMODAL_REGISTRY
|
||||
info = mm_registry.get_processing_info(self.model_config)
|
||||
assert isinstance(info, Qwen2VLProcessingInfo)
|
||||
max_frames_per_video = info.get_num_frames_with_most_features(
|
||||
seq_len=self.model_config.max_model_len,
|
||||
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
|
||||
|
||||
@@ -286,11 +286,14 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
|
||||
|
||||
def get_replacement_qwen2_audio(item_idx: int):
|
||||
if "audio_num_tokens" in out_mm_data:
|
||||
num_features = int(out_mm_data["audio_num_tokens"][item_idx])
|
||||
audio_num_tokens = out_mm_data["audio_num_tokens"]
|
||||
assert isinstance(audio_num_tokens, torch.Tensor)
|
||||
num_features = int(audio_num_tokens[item_idx])
|
||||
elif audio_output_lengths:
|
||||
num_features = audio_output_lengths[item_idx]
|
||||
else:
|
||||
audio_embeds = out_mm_data["audio_embeds"][item_idx]
|
||||
assert isinstance(audio_embeds, torch.Tensor)
|
||||
assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor"
|
||||
num_features = audio_embeds.shape[0]
|
||||
|
||||
|
||||
@@ -153,6 +153,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
|
||||
prefix=f"{prefix}.shared_expert_gate",
|
||||
)
|
||||
|
||||
self.shared_expert: Qwen2MoeMLP | None
|
||||
if config.shared_expert_intermediate_size > 0:
|
||||
self.shared_expert = Qwen2MoeMLP(
|
||||
hidden_size=config.hidden_size,
|
||||
@@ -252,6 +253,12 @@ class Qwen2MoeAttention(nn.Module):
|
||||
rope_parameters=rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
)
|
||||
attention_kwargs: dict[str, Any] = {}
|
||||
if dual_chunk_attention_config:
|
||||
attention_kwargs = {
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -260,12 +267,7 @@ class Qwen2MoeAttention(nn.Module):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
**{
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
if dual_chunk_attention_config
|
||||
else {},
|
||||
**attention_kwargs,
|
||||
)
|
||||
|
||||
def forward(
|
||||
|
||||
@@ -1211,17 +1211,23 @@ class Qwen2VLForConditionalGeneration(
|
||||
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
offset = mm_feature.mm_position.offset
|
||||
data = mm_feature.data
|
||||
assert data is not None
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
image_grid_thw = data["image_grid_thw"]
|
||||
assert isinstance(image_grid_thw.data, torch.Tensor)
|
||||
t, h, w = image_grid_thw.data.tolist()
|
||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||
yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
|
||||
elif mm_feature.modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
video_grid_thw = data["video_grid_thw"]
|
||||
assert isinstance(video_grid_thw.data, torch.Tensor)
|
||||
t, h, w = video_grid_thw.data.tolist()
|
||||
second_per_grid_ts = 1.0
|
||||
if mm_feature.data.get("second_per_grid_ts", None):
|
||||
second_per_grid_ts = mm_feature.data[
|
||||
"second_per_grid_ts"
|
||||
].data.item()
|
||||
second_per_grid_ts_field = data.get("second_per_grid_ts")
|
||||
if second_per_grid_ts_field is not None:
|
||||
assert isinstance(second_per_grid_ts_field.data, torch.Tensor)
|
||||
second_per_grid_ts = second_per_grid_ts_field.data.item()
|
||||
t_factor = second_per_grid_ts * tokens_per_second
|
||||
yield (
|
||||
offset,
|
||||
@@ -1285,7 +1291,7 @@ class Qwen2VLForConditionalGeneration(
|
||||
super().__init__()
|
||||
config: Qwen2VLConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
self.model_config = vllm_config.model_config
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.config = config
|
||||
@@ -1328,7 +1334,7 @@ class Qwen2VLForConditionalGeneration(
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
if image_embeds is not None:
|
||||
else:
|
||||
return Qwen2VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds,
|
||||
@@ -1352,7 +1358,7 @@ class Qwen2VLForConditionalGeneration(
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
else:
|
||||
return Qwen2VLVideoEmbeddingInputs(
|
||||
type="video_embeds",
|
||||
video_embeds=video_embeds,
|
||||
@@ -1411,7 +1417,7 @@ class Qwen2VLForConditionalGeneration(
|
||||
return video_embeds.split(sizes)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
modalities = {}
|
||||
modalities: dict[str, Qwen2VLImageInputs | Qwen2VLVideoInputs | None] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -1481,6 +1487,7 @@ class Qwen2VLForConditionalGeneration(
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
mm_registry = MULTIMODAL_REGISTRY
|
||||
info = mm_registry.get_processing_info(self.model_config)
|
||||
assert isinstance(info, Qwen2VLProcessingInfo)
|
||||
max_frames_per_video = info.get_num_frames_with_most_features(
|
||||
seq_len=self.model_config.max_model_len,
|
||||
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
|
||||
|
||||
@@ -130,6 +130,12 @@ class Qwen3Attention(nn.Module):
|
||||
if attn_type == AttentionType.ENCODER_ONLY
|
||||
else Attention
|
||||
)
|
||||
attention_kwargs: dict[str, Any] = {}
|
||||
if dual_chunk_attention_config:
|
||||
attention_kwargs = {
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
self.attn = attn_cls(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -140,12 +146,7 @@ class Qwen3Attention(nn.Module):
|
||||
per_layer_sliding_window=per_layer_sliding_window,
|
||||
prefix=f"{prefix}.attn",
|
||||
attn_type=attn_type,
|
||||
**{
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
if dual_chunk_attention_config
|
||||
else {},
|
||||
**attention_kwargs,
|
||||
)
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
|
||||
|
||||
from collections.abc import Iterable
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from torch import nn
|
||||
@@ -54,6 +55,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding,
|
||||
)
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.multimodal.inputs import MultiModalFeatureSpec
|
||||
from vllm.sequence import IntermediateTensors
|
||||
from vllm.tokenizers.registry import cached_tokenizer_from_config
|
||||
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
|
||||
@@ -442,7 +444,7 @@ class Qwen3_5ForCausalLMBase(
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
input_tokens: list[int],
|
||||
mm_features: list[object],
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
) -> tuple[torch.Tensor, int]:
|
||||
positions = torch.arange(len(input_tokens), dtype=torch.long)
|
||||
return positions.unsqueeze(0).expand(3, -1), 0
|
||||
@@ -488,7 +490,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
nn.Module.__init__(self)
|
||||
config: Qwen3_5Config = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
@@ -648,6 +650,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
|
||||
|
||||
|
||||
class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
|
||||
language_model: Qwen3_5ForCausalLM
|
||||
num_local_physical_experts: int
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
@@ -700,14 +705,14 @@ class Qwen3_5MoeForConditionalGeneration(
|
||||
Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
|
||||
):
|
||||
# For MoE LoRA weights loading
|
||||
is_3d_moe_weight: bool = True
|
||||
is_3d_moe_weight: ClassVar[bool] = True
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
|
||||
# protocols have not __init__ method, so we need to use nn.Module.__init__
|
||||
nn.Module.__init__(self)
|
||||
config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
|
||||
@@ -286,7 +286,7 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
|
||||
|
||||
return inputs_embeds
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -274,7 +274,7 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
|
||||
|
||||
|
||||
class Qwen3ASRMultiModalProcessor(
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor,
|
||||
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
|
||||
):
|
||||
def _get_mm_fields_config(
|
||||
self,
|
||||
@@ -316,6 +316,7 @@ class Qwen3ASRMultiModalProcessor(
|
||||
if num_features == 0:
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
audio = audios.get(item_idx)
|
||||
assert audio is not None
|
||||
raise ValueError(
|
||||
f"The audio {audio} (len={len(audio)}) is too short "
|
||||
"to be represented inside the model"
|
||||
|
||||
@@ -72,8 +72,9 @@ class Qwen3ASRForcedAlignerForTokenClassification(
|
||||
|
||||
# Remove the unused generation head created by the base class;
|
||||
# the forced aligner uses a classifier head instead.
|
||||
self.language_model.lm_head = None
|
||||
self.language_model.logits_processor = None
|
||||
language_model: nn.Module = self.language_model
|
||||
language_model.lm_head = None
|
||||
language_model.logits_processor = None
|
||||
|
||||
self.classify_num = thinker_config.classify_num
|
||||
|
||||
|
||||
@@ -120,6 +120,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
|
||||
)
|
||||
|
||||
audio_data = audios[0]
|
||||
assert audio_data is not None
|
||||
audio_feature_lengths = audio_data.get("audio_feature_lengths")
|
||||
if audio_feature_lengths is not None:
|
||||
if isinstance(audio_feature_lengths.data, torch.Tensor):
|
||||
|
||||
@@ -92,7 +92,9 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
|
||||
if override is not None:
|
||||
return bool(override)
|
||||
layer_types = getattr(config, "layer_types", None)
|
||||
return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION
|
||||
if not layer_types:
|
||||
return False
|
||||
return layer_types[layer_idx] == _SLIDING_ATTENTION
|
||||
|
||||
|
||||
def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||
@@ -105,6 +107,7 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
|
||||
|
||||
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
|
||||
spec_config = vllm_config.speculative_config
|
||||
assert spec_config is not None
|
||||
config = spec_config.draft_model_config.hf_config
|
||||
aux_layers = get_eagle3_aux_layers_from_config(spec_config)
|
||||
num_features_to_use = len(aux_layers) if aux_layers else config.num_hidden_layers
|
||||
@@ -418,7 +421,9 @@ class DFlashQwen3Model(nn.Module):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
self.config = speculative_config.draft_model_config.hf_config
|
||||
self.vocab_size = self.config.vocab_size
|
||||
self.quant_config = get_draft_quant_config(vllm_config)
|
||||
_add_global_draft_layer_exclusions(
|
||||
@@ -759,7 +764,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
self.draft_model_config = speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
@@ -801,7 +808,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -896,7 +903,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
model_weights["model.mask_embedding"] = mask_embedding
|
||||
self.model.has_separate_mask_embedding = True
|
||||
|
||||
orig_to_new_substr = {}
|
||||
orig_to_new_substr: dict[str, None] = {}
|
||||
if not includes_draft_id_mapping:
|
||||
orig_to_new_substr["draft_id_to_target_id"] = None
|
||||
if not includes_embed_tokens:
|
||||
|
||||
@@ -233,7 +233,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
|
||||
class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
|
||||
nn.Module.__init__(self)
|
||||
self.draft_model_config = vllm_config.speculative_config.draft_model_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
self.draft_model_config = speculative_config.draft_model_config
|
||||
self.config = self.draft_model_config.hf_config
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
|
||||
|
||||
@@ -101,7 +101,7 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
|
||||
hidden_states = self.hidden_norm(hidden_states)
|
||||
return hidden_states, residual
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
embeds: torch.Tensor,
|
||||
@@ -159,7 +159,9 @@ class Qwen3Eagle3Model(nn.Module):
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
self.config = speculative_config.draft_model_config.hf_config
|
||||
self.vocab_size = self.config.vocab_size
|
||||
|
||||
# Get drafter's quantization config
|
||||
@@ -210,6 +212,7 @@ class Qwen3Eagle3Model(nn.Module):
|
||||
self.config, "target_hidden_size", self.config.hidden_size
|
||||
)
|
||||
self.fc_input_size = target_hidden_size * num_aux_features
|
||||
self.input_norm: RMSNorm | None
|
||||
if self.norm_before_fc:
|
||||
self.input_norm = RMSNorm(
|
||||
self.fc_input_size,
|
||||
@@ -282,7 +285,9 @@ class Qwen3Eagle3Model(nn.Module):
|
||||
class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
self.config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
self.config = speculative_config.draft_model_config.hf_config
|
||||
# Ensure draft_vocab_size is set
|
||||
# default to the base vocab size when absent
|
||||
if getattr(self.config, "draft_vocab_size", None) is None:
|
||||
@@ -318,7 +323,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
else:
|
||||
self.draft_id_to_target_id = None
|
||||
|
||||
self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting
|
||||
self.use_parallel_drafting = speculative_config.parallel_drafting
|
||||
|
||||
if self.use_parallel_drafting:
|
||||
self.register_buffer(
|
||||
@@ -335,7 +340,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
|
||||
) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
|
||||
@@ -177,6 +177,8 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
|
||||
shared_expert_intermediate_size = getattr(
|
||||
config, "shared_expert_intermediate_size", 0
|
||||
)
|
||||
self.shared_expert_gate: ReplicatedLinear | None
|
||||
self.shared_expert: Qwen3MoeMLP | None
|
||||
if shared_expert_intermediate_size > 0:
|
||||
self.shared_expert_gate = ReplicatedLinear(
|
||||
config.hidden_size,
|
||||
@@ -302,6 +304,12 @@ class Qwen3MoeAttention(nn.Module):
|
||||
rope_parameters=rope_parameters,
|
||||
dual_chunk_attention_config=dual_chunk_attention_config,
|
||||
)
|
||||
attention_kwargs: dict[str, Any] = {}
|
||||
if dual_chunk_attention_config:
|
||||
attention_kwargs = {
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
self.attn = Attention(
|
||||
self.num_heads,
|
||||
self.head_dim,
|
||||
@@ -310,12 +318,7 @@ class Qwen3MoeAttention(nn.Module):
|
||||
cache_config=cache_config,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.attn",
|
||||
**{
|
||||
"layer_idx": extract_layer_index(prefix),
|
||||
"dual_chunk_attention_config": dual_chunk_attention_config,
|
||||
}
|
||||
if dual_chunk_attention_config
|
||||
else {},
|
||||
**attention_kwargs,
|
||||
)
|
||||
|
||||
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
|
||||
|
||||
@@ -757,6 +757,9 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
|
||||
|
||||
|
||||
class QwenNextMixtureOfExperts(MixtureOfExperts):
|
||||
model: Qwen3NextModel
|
||||
num_local_physical_experts: int
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
|
||||
@@ -24,7 +24,7 @@
|
||||
|
||||
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
|
||||
from functools import partial
|
||||
from typing import Any, cast
|
||||
from typing import Any, ClassVar
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -41,12 +41,13 @@ from transformers.models.qwen3_omni_moe.processing_qwen3_omni_moe import (
|
||||
Qwen3OmniMoeProcessor,
|
||||
)
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
from typing_extensions import TypeVar
|
||||
|
||||
from vllm.compilation.decorators import support_torch_compile
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
|
||||
from vllm.inputs import PromptType
|
||||
from vllm.inputs import PromptType, TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
|
||||
from vllm.model_executor.layers.attention.mm_encoder_attention import (
|
||||
@@ -73,6 +74,7 @@ from vllm.multimodal.inputs import (
|
||||
)
|
||||
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
|
||||
from vllm.multimodal.processing.processor import (
|
||||
BaseProcessingInfo,
|
||||
HFMultiModalInputs,
|
||||
MultiModalProcessingResult,
|
||||
MultiModalPromptUpdates,
|
||||
@@ -102,12 +104,15 @@ from .qwen2_5_omni_thinker import (
|
||||
Qwen2_5OmniThinkerDummyInputsBuilder,
|
||||
Qwen2_5OmniThinkerMultiModalDataParser,
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
_get_feature_tensor,
|
||||
check_interleaved_audio_video,
|
||||
merge_interleaved_embeddings,
|
||||
)
|
||||
from .qwen2_5_vl import (
|
||||
Qwen2_5_VisionAttention,
|
||||
Qwen2_5_VLImageInputs,
|
||||
Qwen2_5_VLProcessingInfo,
|
||||
Qwen2_5_VLVideoInputs,
|
||||
)
|
||||
from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel
|
||||
from .utils import (
|
||||
@@ -963,7 +968,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
grid_thw: list[list[int]],
|
||||
grid_thw: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
hidden_states = x.to(device=self.device, dtype=self.dtype)
|
||||
hidden_states = self.patch_embed(hidden_states)
|
||||
@@ -1201,7 +1206,7 @@ class Qwen3OmniMoeThinkerProcessingInfo(
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int] | None = None,
|
||||
) -> Mapping[str, int] | None:
|
||||
) -> Mapping[str, int]:
|
||||
mm_counts = mm_counts or {}
|
||||
requested_modalities = {m for m, c in mm_counts.items() if c > 0}
|
||||
mm_max_tokens: dict[str, int] = {}
|
||||
@@ -1231,11 +1236,15 @@ class Qwen3OmniMoeThinkerProcessingInfo(
|
||||
return mm_max_tokens
|
||||
|
||||
|
||||
Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
|
||||
Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder[
|
||||
Qwen3OmniMoeThinkerProcessingInfo
|
||||
]
|
||||
|
||||
_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen3OmniMoeThinkerProcessingInfo)
|
||||
|
||||
|
||||
class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
Qwen2_5OmniThinkerMultiModalProcessor,
|
||||
Qwen2_5OmniThinkerMultiModalProcessor[_I],
|
||||
):
|
||||
def _get_hf_mm_inputs(
|
||||
self,
|
||||
@@ -1246,7 +1255,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
|
||||
audios = hf_inputs.hf_data.get("audio")
|
||||
if audios:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
assert isinstance(audios, (list, tuple))
|
||||
feature_extractor = self.info.get_hf_processor().feature_extractor
|
||||
hop_length = feature_extractor.hop_length
|
||||
|
||||
def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
|
||||
@@ -1283,9 +1293,10 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
):
|
||||
audios = mm_data.get("audio", [])
|
||||
if audios:
|
||||
feature_extractor = self.info.get_feature_extractor(
|
||||
assert isinstance(audios, (list, tuple))
|
||||
feature_extractor = self.info.get_hf_processor(
|
||||
**hf_processor_mm_kwargs
|
||||
)
|
||||
).feature_extractor
|
||||
hop_length = feature_extractor.hop_length
|
||||
|
||||
audio_num_frames = []
|
||||
@@ -1421,6 +1432,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
) -> Sequence[PromptUpdate]:
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
tokenizer = self.info.get_tokenizer()
|
||||
assert isinstance(self.info, Qwen2_5_VLProcessingInfo)
|
||||
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
|
||||
vocab = tokenizer.get_vocab()
|
||||
|
||||
@@ -1460,6 +1472,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
|
||||
if num_features == 0:
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
audio = audios.get(item_idx)
|
||||
assert audio is not None
|
||||
raise ValueError(
|
||||
f"The audio {audio} (len={len(audio)}) is too short "
|
||||
"to be represented inside the model"
|
||||
@@ -1607,8 +1620,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
Qwen3OmniMoeConditionalGenerationMixin,
|
||||
SupportsTranscription,
|
||||
):
|
||||
is_3d_moe_weight: bool = True
|
||||
supports_tower_connector_lora: bool = True
|
||||
is_3d_moe_weight: ClassVar[bool] = True
|
||||
supports_tower_connector_lora: ClassVar[bool] = True
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
"thinker.lm_head.": "language_model.lm_head.",
|
||||
@@ -1769,7 +1782,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
self.deepstack_input_embeds_num_tokens = 0
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[
|
||||
str,
|
||||
Qwen2_5_VLImageInputs
|
||||
| Qwen2_5_VLVideoInputs
|
||||
| Qwen2_5OmniAudioFeatureInputs
|
||||
| None,
|
||||
] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -1862,6 +1881,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
if self.visual.deepstack_visual_indexes is not None and any(
|
||||
has_vision_embeddings
|
||||
):
|
||||
multimodal_embeddings = list(multimodal_embeddings)
|
||||
multiscale_len = len(self.visual.deepstack_visual_indexes)
|
||||
multimodal_embeddings_multiscale = []
|
||||
|
||||
@@ -2006,8 +2026,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
f
|
||||
for f in mm_features
|
||||
if f.modality == "video"
|
||||
and f.data is not None
|
||||
and f.data.get("use_audio_in_video")
|
||||
and f.data["use_audio_in_video"].data.item()
|
||||
and _get_feature_tensor(f, "use_audio_in_video").item()
|
||||
]
|
||||
audios = [f for f in mm_features if f.modality == "audio"]
|
||||
|
||||
@@ -2015,7 +2036,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
paired_audio_offsets: set[int] = set()
|
||||
for i, video_f in enumerate(videos_with_audio):
|
||||
if i < len(audios):
|
||||
audio_len = audios[i].data["audio_feature_lengths"].data.item()
|
||||
audio_len = int(
|
||||
_get_feature_tensor(audios[i], "audio_feature_lengths").item()
|
||||
)
|
||||
mapping[video_f.mm_position.offset] = audio_len
|
||||
paired_audio_offsets.add(audios[i].mm_position.offset)
|
||||
return mapping, paired_audio_offsets
|
||||
@@ -2043,9 +2066,10 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
for mm_feature in sorted_features:
|
||||
offset = mm_feature.mm_position.offset
|
||||
modality = mm_feature.modality
|
||||
assert mm_feature.data is not None
|
||||
|
||||
if modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
|
||||
yield (
|
||||
offset,
|
||||
"image",
|
||||
@@ -2057,15 +2081,15 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
},
|
||||
)
|
||||
elif modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
|
||||
second_per_grid_ts = 2.0
|
||||
if mm_feature.data.get("second_per_grid_ts"):
|
||||
second_per_grid_ts = mm_feature.data[
|
||||
"second_per_grid_ts"
|
||||
].data.item()
|
||||
second_per_grid_ts = _get_feature_tensor(
|
||||
mm_feature, "second_per_grid_ts"
|
||||
).item()
|
||||
use_audio_in_video = bool(
|
||||
mm_feature.data.get("use_audio_in_video")
|
||||
and mm_feature.data["use_audio_in_video"].data.item()
|
||||
and _get_feature_tensor(mm_feature, "use_audio_in_video").item()
|
||||
)
|
||||
|
||||
yield (
|
||||
@@ -2082,7 +2106,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
)
|
||||
elif modality == "audio":
|
||||
if offset not in paired_audio_offsets:
|
||||
audio_len = mm_feature.data["audio_feature_lengths"].data.item()
|
||||
audio_len = _get_feature_tensor(
|
||||
mm_feature, "audio_feature_lengths"
|
||||
).item()
|
||||
yield offset, "audio", {"audio_feature_length": audio_len}
|
||||
|
||||
def _compute_interleaved_positions(
|
||||
@@ -2170,8 +2196,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
to_language = "en"
|
||||
|
||||
# Get full language names from supported_languages mapping
|
||||
full_lang_name = cls.supported_languages.get(language, "")
|
||||
full_lang_name_to = cls.supported_languages.get(to_language, "")
|
||||
full_lang_name = (
|
||||
cls.supported_languages.get(language, "") if language is not None else ""
|
||||
)
|
||||
full_lang_name_to = (
|
||||
cls.supported_languages.get(to_language, "")
|
||||
if to_language is not None
|
||||
else ""
|
||||
)
|
||||
|
||||
if task_type == "transcribe" and full_lang_name:
|
||||
instruction += f" into {full_lang_name}"
|
||||
@@ -2201,8 +2233,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
|
||||
)
|
||||
|
||||
audio_data = (audio, stt_config.sample_rate)
|
||||
prompts_dict = {"multi_modal_data": {"audio": audio_data}, "prompt": prompt}
|
||||
return cast(PromptType, prompts_dict)
|
||||
prompts_dict: TextPrompt = {
|
||||
"multi_modal_data": {"audio": audio_data},
|
||||
"prompt": prompt,
|
||||
}
|
||||
return prompts_dict
|
||||
|
||||
def get_mrope_input_positions(
|
||||
self,
|
||||
|
||||
@@ -582,7 +582,9 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
else []
|
||||
)
|
||||
self.num_grid_per_side = int(self.num_position_embeddings**0.5)
|
||||
self._rot_pos_ids_cache = LRUCache(capacity=1024)
|
||||
self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = (
|
||||
LRUCache(capacity=1024)
|
||||
)
|
||||
|
||||
use_data_parallel = is_vit_use_data_parallel()
|
||||
self.tp_size = (
|
||||
@@ -705,7 +707,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
self._rot_pos_ids_cache[cache_key] = result
|
||||
return result
|
||||
|
||||
def rot_pos_emb(self, grid_thw: list[list[int]]):
|
||||
def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]):
|
||||
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
|
||||
pos_ids = [
|
||||
self.rot_pos_ids(h, w, self.spatial_merge_size)
|
||||
@@ -731,7 +733,9 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
|
||||
return cos_combined, sin_combined
|
||||
|
||||
def fast_pos_embed_interpolate(self, grid_thw: list[list[int]]) -> torch.Tensor:
|
||||
def fast_pos_embed_interpolate(
|
||||
self, grid_thw: Sequence[Sequence[int]]
|
||||
) -> torch.Tensor:
|
||||
interpolate_fn = (
|
||||
triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native
|
||||
)
|
||||
@@ -752,7 +756,7 @@ class Qwen3_VisionTransformer(nn.Module):
|
||||
|
||||
def prepare_encoder_metadata(
|
||||
self,
|
||||
grid_thw_list: list[list[int]],
|
||||
grid_thw_list: Sequence[Sequence[int]],
|
||||
*,
|
||||
max_batch_size: int | None = None,
|
||||
max_frames_per_batch: int | None = None,
|
||||
@@ -1005,9 +1009,10 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES,
|
||||
) -> int:
|
||||
return super().get_num_frames_with_most_features(
|
||||
seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES
|
||||
seq_len, mm_counts, max_frames_per_video=max_frames_per_video
|
||||
)
|
||||
|
||||
def get_max_video_tokens(
|
||||
@@ -1253,7 +1258,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
|
||||
)
|
||||
videos = [v.copy() for v in videos]
|
||||
|
||||
video_items = []
|
||||
video_items: list[VideoItem] = []
|
||||
for video in videos:
|
||||
video_num_frames = video.shape[0]
|
||||
video_metadata = {
|
||||
@@ -1339,6 +1344,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
# are processed into several image patches
|
||||
video_input_ids_lst: list[list[int]] = []
|
||||
if videos := hf_data.pop("videos", []):
|
||||
assert isinstance(videos, list)
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
timestamps_per_video = []
|
||||
@@ -1361,7 +1367,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
|
||||
# NOTE: a copy of is created to update do_sample_frames,
|
||||
# otherwise mm_hash for the object will be incorrect.
|
||||
video_mm_kwargs = dict(**hf_kwargs)
|
||||
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
|
||||
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
|
||||
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
|
||||
video_size = dict(self.info.get_video_processor().size)
|
||||
@@ -1551,6 +1557,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
sampled_fps = sampled_fps[item_idx]
|
||||
|
||||
timestamps = out_item["timestamps"].data
|
||||
assert isinstance(timestamps, (list, torch.Tensor))
|
||||
assert len(timestamps) == grid_thw[0], (
|
||||
f"The timestamps length({len(timestamps)}) should be equal "
|
||||
f"video length ({grid_thw[0]})."
|
||||
@@ -1617,7 +1624,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
|
||||
def get_video_repl(
|
||||
*,
|
||||
tokens_per_frame: list[int],
|
||||
timestamps: list[float | int] | torch.Tensor,
|
||||
timestamps: Sequence[float] | torch.Tensor,
|
||||
tokenizer: TokenizerLike,
|
||||
vision_start_token_id: int,
|
||||
vision_end_token_id: int,
|
||||
@@ -1842,7 +1849,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
super().__init__()
|
||||
config: Qwen3VLConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
@@ -2006,6 +2013,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
def get_max_frames_per_video(self) -> int:
|
||||
mm_registry = MULTIMODAL_REGISTRY
|
||||
info = mm_registry.get_processing_info(self.model_config)
|
||||
assert isinstance(info, Qwen3VLProcessingInfo)
|
||||
max_frames_per_video = info.get_num_frames_with_most_features(
|
||||
seq_len=self.model_config.max_model_len,
|
||||
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
|
||||
@@ -2254,7 +2262,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
|
||||
if image_embeds is not None:
|
||||
else:
|
||||
return Qwen2_5_VLImageEmbeddingInputs(
|
||||
type="image_embeds",
|
||||
image_embeds=image_embeds,
|
||||
@@ -2282,7 +2290,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
timestamps=timestamps,
|
||||
)
|
||||
|
||||
if video_embeds is not None:
|
||||
else:
|
||||
return Qwen2_5_VLVideoEmbeddingInputs(
|
||||
type="video_embeds",
|
||||
video_embeds=video_embeds,
|
||||
@@ -2405,6 +2413,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
# Apply EVS to each video.
|
||||
video_embeds_out = []
|
||||
assert video_input.timestamps is not None
|
||||
for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)):
|
||||
# Compute positions.
|
||||
timestamps = video_input.timestamps[video_idx]
|
||||
@@ -2412,6 +2421,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
|
||||
t, h, w = size
|
||||
if self.is_multimodal_pruning_enabled:
|
||||
assert self.video_pruning_rate is not None
|
||||
# Compute the retention mask for each video (EVS or VidCom2).
|
||||
if self.video_pruning_method == "vidcom2":
|
||||
mask_fn = vidcom2_compute_retention_mask
|
||||
@@ -2583,7 +2593,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
{
|
||||
"video_grid_thw": MultiModalFieldElem(
|
||||
data=torch.tensor(video_grid_thw),
|
||||
field=None, # HACK.
|
||||
field=MultiModalFieldConfig.batched("video").field,
|
||||
),
|
||||
}
|
||||
),
|
||||
@@ -2611,7 +2621,9 @@ class Qwen3VLForConditionalGeneration(
|
||||
return expanded_positions
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[
|
||||
str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
|
||||
] = {}
|
||||
for input_key in kwargs:
|
||||
if (
|
||||
input_key in ("pixel_values", "image_embeds")
|
||||
@@ -2659,14 +2671,20 @@ class Qwen3VLForConditionalGeneration(
|
||||
"""
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
offset = mm_feature.mm_position.offset
|
||||
data = mm_feature.data
|
||||
assert data is not None
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
grid = data["image_grid_thw"].data
|
||||
assert isinstance(grid, torch.Tensor)
|
||||
t, h, w = grid.tolist()
|
||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||
llm_grid_h = h // spatial_merge_size
|
||||
llm_grid_w = w // spatial_merge_size
|
||||
yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w
|
||||
elif mm_feature.modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
grid = data["video_grid_thw"].data
|
||||
assert isinstance(grid, torch.Tensor)
|
||||
t, h, w = grid.tolist()
|
||||
llm_grid_h = h // spatial_merge_size
|
||||
llm_grid_w = w // spatial_merge_size
|
||||
|
||||
@@ -2721,7 +2739,7 @@ class Qwen3VLForConditionalGeneration(
|
||||
mm_features: list[MultiModalFeatureSpec],
|
||||
config: Qwen3VLConfig,
|
||||
):
|
||||
llm_pos_ids_list = []
|
||||
llm_pos_ids_list: list[np.ndarray] = []
|
||||
st = 0
|
||||
for (
|
||||
offset,
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
"""Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights."""
|
||||
|
||||
from itertools import islice
|
||||
from typing import ClassVar
|
||||
|
||||
import torch
|
||||
from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import (
|
||||
@@ -176,6 +177,9 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
|
||||
|
||||
|
||||
class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
|
||||
language_model: Qwen3MoeLLMForCausalLM
|
||||
num_local_physical_experts: int
|
||||
|
||||
def update_physical_experts_metadata(
|
||||
self,
|
||||
num_physical_experts: int,
|
||||
@@ -223,7 +227,7 @@ class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
|
||||
class Qwen3VLMoeForConditionalGeneration(
|
||||
Qwen3VLForConditionalGeneration, Qwen3VLMoeMixtureOfExperts
|
||||
):
|
||||
is_3d_moe_weight: bool = True
|
||||
is_3d_moe_weight: ClassVar[bool] = True
|
||||
packed_modules_mapping = {
|
||||
"qkv_proj": [
|
||||
"q_proj",
|
||||
@@ -236,7 +240,7 @@ class Qwen3VLMoeForConditionalGeneration(
|
||||
super(Qwen3VLForConditionalGeneration, self).__init__()
|
||||
config: Qwen3VLMoeConfig = vllm_config.model_config.hf_config
|
||||
quant_config = vllm_config.quant_config
|
||||
multimodal_config = vllm_config.model_config.multimodal_config
|
||||
multimodal_config = vllm_config.model_config.get_multimodal_config()
|
||||
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
@@ -293,8 +297,8 @@ class Qwen3VLMoeForConditionalGeneration(
|
||||
)
|
||||
|
||||
# Whether to include the gate_up_proj mapping is determined by
|
||||
# the language model.
|
||||
self.packed_modules_mapping = (
|
||||
# the language model. Keep this override local to the instance.
|
||||
self.packed_modules_mapping = ( # type: ignore[misc]
|
||||
self.packed_modules_mapping | self.language_model.packed_modules_mapping
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user