diff --git a/tools/pre_commit/mypy.py b/tools/pre_commit/mypy.py index 71dd929d37..f9d064a499 100755 --- a/tools/pre_commit/mypy.py +++ b/tools/pre_commit/mypy.py @@ -113,7 +113,6 @@ SEPARATE_GROUPS = [ EXCLUDE = [ r"vllm/model_executor/models/[kK]", - r"vllm/model_executor/models/[qQ]", r"vllm/model_executor/models/[uU]", r"vllm/model_executor/models/[vV]", r"vllm/model_executor/models/[wW]", diff --git a/vllm/model_executor/models/interfaces.py b/vllm/model_executor/models/interfaces.py index 21a09cb7ba..7901a136f5 100644 --- a/vllm/model_executor/models/interfaces.py +++ b/vllm/model_executor/models/interfaces.py @@ -1262,7 +1262,7 @@ class SupportsRealtime(Protocol): Override in subclasses based on the model's expected output length.""" @classmethod - async def buffer_realtime_audio( + def buffer_realtime_audio( cls, audio_stream: AsyncGenerator[np.ndarray, None], input_stream: asyncio.Queue[list[int]], diff --git a/vllm/model_executor/models/qianfan_ocr.py b/vllm/model_executor/models/qianfan_ocr.py index 335a3f0f98..23b1092419 100644 --- a/vllm/model_executor/models/qianfan_ocr.py +++ b/vllm/model_executor/models/qianfan_ocr.py @@ -36,14 +36,14 @@ class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo): config = self.get_hf_config() vision_config = config.vision_config - kwargs = self.ctx.get_merged_mm_kwargs(kwargs) - kwargs.setdefault("image_size", vision_config.image_size) - kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) - kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) - kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) - kwargs.setdefault("use_thumbnail", config.use_thumbnail) + merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs) + merged_kwargs.setdefault("image_size", vision_config.image_size) + merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch) + merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch) + merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size) + merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail) - image_processor = InternVLImageProcessor(**kwargs) + image_processor = InternVLImageProcessor(**merged_kwargs) image_size = image_processor.image_size patch_size = vision_config.patch_size downsample_ratio = config.downsample_ratio @@ -73,7 +73,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel): """ def _patch_quant_config( - self, config: PreTrainedConfig, quant_config: QuantizationConfig + self, config: PreTrainedConfig, quant_config: QuantizationConfig | None ) -> None: super()._patch_quant_config(config, quant_config) # ignore vit layers to preserve model performance diff --git a/vllm/model_executor/models/qwen2.py b/vllm/model_executor/models/qwen2.py index 7513936ebb..88600b3767 100644 --- a/vllm/model_executor/models/qwen2.py +++ b/vllm/model_executor/models/qwen2.py @@ -187,6 +187,12 @@ class Qwen2Attention(nn.Module): if attn_type == AttentionType.ENCODER_ONLY else Attention ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = attn_cls( self.num_heads, self.head_dim, @@ -196,12 +202,7 @@ class Qwen2Attention(nn.Module): quant_config=quant_config, attn_type=attn_type, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) def forward( diff --git a/vllm/model_executor/models/qwen2_5_omni_thinker.py b/vllm/model_executor/models/qwen2_5_omni_thinker.py index 0ce2c37313..986de3e096 100644 --- a/vllm/model_executor/models/qwen2_5_omni_thinker.py +++ b/vllm/model_executor/models/qwen2_5_omni_thinker.py @@ -42,6 +42,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import ( Qwen2_5OmniProcessor, ) from transformers.models.whisper import WhisperFeatureExtractor +from typing_extensions import TypeVar from vllm.config import VllmConfig from vllm.config.multimodal import MultiModalDummyOptions @@ -63,7 +64,10 @@ from vllm.model_executor.models.qwen2_audio import ( Qwen2AudioProcessingInfo, _get_feat_extract_output_lengths, ) -from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser +from vllm.model_executor.models.qwen2_vl import ( + Qwen2VLMultiModalDataParser, + Qwen2VLProcessingInfo, +) from vllm.multimodal import MULTIMODAL_REGISTRY from vllm.multimodal.inputs import ( ImageItem, @@ -79,6 +83,7 @@ from vllm.multimodal.parse import ( ) from vllm.multimodal.processing import ( BaseDummyInputsBuilder, + BaseProcessingInfo, ) from vllm.multimodal.processing.processor import ( BaseMultiModalProcessor, @@ -118,6 +123,14 @@ except (ImportError, ModuleNotFoundError): logger = init_logger(__name__) +def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tensor: + data = feature.data + assert data is not None + tensor = data[key].data + assert isinstance(tensor, torch.Tensor) + return tensor + + def unpad_and_flat_audio_features( input_audio_features: torch.Tensor, audio_feature_lengths: torch.Tensor, @@ -347,7 +360,7 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser): def _parse_audio_data( self, data: dict[str, torch.Tensor] | ModalityData[ImageItem], - ) -> ModalityDataItems[Any, Any]: + ) -> ModalityDataItems[Any, Any] | None: if isinstance(data, dict): required, optional = self.embedding_field_sets("audio") data = dict(data) @@ -406,7 +419,7 @@ class Qwen2_5OmniThinkerProcessingInfo( self, seq_len: int, mm_counts: Mapping[str, int] | None = None, - ) -> Mapping[str, int] | None: + ) -> Mapping[str, int]: mm_counts = mm_counts or {} requested_modalities = {m for m, c in mm_counts.items() if c > 0} mm_max_tokens: dict[str, int] = {} @@ -436,9 +449,13 @@ class Qwen2_5OmniThinkerProcessingInfo( return mm_max_tokens -class Qwen2_5OmniThinkerDummyInputsBuilder( - BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo] -): +_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo) +_VI = TypeVar( + "_VI", bound=Qwen2VLProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo +) + + +class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[_VI]): def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_audios = mm_counts.get("audio", 0) num_images = mm_counts.get("image", 0) @@ -462,7 +479,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( mm_counts: Mapping[str, int], mm_options: MultiModalDummyOptions, ) -> MultiModalDataDict: - feature_extractor = self.info.get_feature_extractor() + feature_extractor = self.info.get_hf_processor().feature_extractor target_audio_length = ( min( @@ -501,9 +518,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder( return mm_data -class Qwen2_5OmniThinkerMultiModalProcessor( - BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo] -): +class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]): def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str: return self.dummy_inputs.get_dummy_text(mm_counts) @@ -651,6 +666,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( for item in mm_kwargs["video"]: if item and item.get("use_audio_in_video"): use_audio_in_video_tensor = item["use_audio_in_video"].data + assert isinstance(use_audio_in_video_tensor, torch.Tensor) if use_audio_in_video_tensor.numel() > 0: use_audio_in_video = bool(use_audio_in_video_tensor.item()) break @@ -749,6 +765,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( ) -> Sequence[PromptUpdate]: processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = self.info.get_tokenizer() + assert isinstance(self.info, Qwen2VLProcessingInfo) image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) vocab = tokenizer.get_vocab() @@ -786,6 +803,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor( if num_features == 0: audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) + assert audio is not None raise ValueError( f"The audio {audio} (len={len(audio)}) is too short " "to be represented inside the model" @@ -887,6 +905,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor( class Qwen2_5OmniConditionalGenerationMixin: + audio_tower: nn.Module + visual: nn.Module + vllm_config: VllmConfig + def _parse_and_validate_audio_input( self, **kwargs: object ) -> Qwen2_5OmniAudioFeatureInputs | None: @@ -916,7 +938,7 @@ class Qwen2_5OmniConditionalGenerationMixin: def _parse_and_validate_image_input( self, - **kwargs: dict[str, Any], + **kwargs: object, ) -> Qwen2_5_VLImageInputs | None: pixel_values = kwargs.pop("pixel_values", None) image_embeds = kwargs.pop("image_embeds", None) @@ -932,7 +954,7 @@ class Qwen2_5OmniConditionalGenerationMixin: image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2_5_VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -941,7 +963,7 @@ class Qwen2_5OmniConditionalGenerationMixin: def _parse_and_validate_video_input( self, - **kwargs: dict[str, Any], + **kwargs: object, ) -> Qwen2_5_VLVideoInputs | None: pixel_values_videos = kwargs.pop("pixel_values_videos", None) video_embeds = kwargs.pop("video_embeds", None) @@ -957,7 +979,7 @@ class Qwen2_5OmniConditionalGenerationMixin: video_grid_thw=video_grid_thw, ) - if video_embeds is not None: + else: if not isinstance(video_embeds, torch.Tensor): raise ValueError( "Incorrect type of video embeddings. " @@ -1130,7 +1152,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration( ) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, + Qwen2_5_VLImageInputs + | Qwen2_5_VLVideoInputs + | Qwen2_5OmniAudioFeatureInputs + | None, + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1175,8 +1203,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration( f for f in mm_features if f.modality == "video" + and f.data is not None and f.data.get("use_audio_in_video") - and f.data["use_audio_in_video"].data.item() + and _get_feature_tensor(f, "use_audio_in_video").item() ] audios = [f for f in mm_features if f.modality == "audio"] @@ -1185,7 +1214,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration( paired_audio_offsets: set[int] = set() for i, video_f in enumerate(videos_with_audio): if i < len(audios): - audio_len = audios[i].data["audio_feature_lengths"].data.item() + audio_len = int( + _get_feature_tensor(audios[i], "audio_feature_lengths").item() + ) mapping[video_f.mm_position.offset] = audio_len paired_audio_offsets.add(audios[i].mm_position.offset) return mapping, paired_audio_offsets @@ -1220,9 +1251,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration( for mm_feature in sorted_features: offset = mm_feature.mm_position.offset modality = mm_feature.modality + assert mm_feature.data is not None if modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist() yield ( offset, "image", @@ -1234,16 +1266,16 @@ class Qwen2_5OmniThinkerForConditionalGeneration( }, ) elif modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist() second_per_grid_ts = 1.0 if mm_feature.data.get("second_per_grid_ts"): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + second_per_grid_ts = _get_feature_tensor( + mm_feature, "second_per_grid_ts" + ).item() use_audio_in_video = False if mm_feature.data.get("use_audio_in_video"): use_audio_in_video = bool( - mm_feature.data["use_audio_in_video"].data.item() + _get_feature_tensor(mm_feature, "use_audio_in_video").item() ) yield ( @@ -1261,7 +1293,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration( elif modality == "audio": # Skip audio that's paired with video (handled in video case) if offset not in paired_audio_offsets: - audio_len = mm_feature.data["audio_feature_lengths"].data.item() + audio_len = _get_feature_tensor( + mm_feature, "audio_feature_lengths" + ).item() yield offset, "audio", {"audio_feature_length": audio_len} def _compute_interleaved_positions( diff --git a/vllm/model_executor/models/qwen2_5_vl.py b/vllm/model_executor/models/qwen2_5_vl.py index 915aaece06..864a77f095 100644 --- a/vllm/model_executor/models/qwen2_5_vl.py +++ b/vllm/model_executor/models/qwen2_5_vl.py @@ -654,7 +654,10 @@ class Qwen2_5_VisionTransformer(nn.Module): self.spatial_merge_size = vision_config.spatial_merge_size self.fullatt_block_indexes = vision_config.fullatt_block_indexes self.spatial_merge_unit = self.spatial_merge_size**2 - self._rope_by_thw_cache = LRUCache(capacity=1024) + self._rope_by_thw_cache: LRUCache[ + tuple[int, int, int], + tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor], + ] = LRUCache(capacity=1024) use_data_parallel = is_vit_use_data_parallel() self.tp_size = ( 1 @@ -885,7 +888,7 @@ class Qwen2_5_VisionTransformer(nn.Module): def prepare_encoder_metadata( self, - grid_thw: list[list[int]], + grid_thw: Sequence[Sequence[int]], *, max_batch_size: int | None = None, max_frames_per_batch: int | None = None, @@ -927,9 +930,9 @@ class Qwen2_5_VisionTransformer(nn.Module): # patchify rotary_pos_emb_cos = [] rotary_pos_emb_sin = [] - window_index: list = [] - cu_window_seqlens: list = [torch.tensor([0], dtype=torch.int32)] - cu_seqlens: list = [] + window_indices: list[torch.Tensor] = [] + cu_window_seqlens_parts = [torch.tensor([0], dtype=torch.int32)] + cu_seqlens_parts: list[torch.Tensor] = [] window_index_id = 0 cu_window_seqlens_last = 0 @@ -946,26 +949,26 @@ class Qwen2_5_VisionTransformer(nn.Module): cu_seqlens_thw, ) = self.get_rope_by_thw(t, h, w) - window_index.append(window_index_thw + window_index_id) + window_indices.append(window_index_thw + window_index_id) window_index_id += t * llm_h * llm_w cu_seqlens_window_thw = cu_seqlens_window_thw + cu_window_seqlens_last cu_window_seqlens_last = cu_seqlens_window_thw[-1] - cu_window_seqlens.append(cu_seqlens_window_thw) + cu_window_seqlens_parts.append(cu_seqlens_window_thw) rotary_pos_emb_cos.append(cos_thw) rotary_pos_emb_sin.append(sin_thw) - cu_seqlens.append(cu_seqlens_thw) + cu_seqlens_parts.append(cu_seqlens_thw) rotary_pos_emb_cos = torch.cat(rotary_pos_emb_cos) rotary_pos_emb_sin = torch.cat(rotary_pos_emb_sin) - window_index = torch.cat(window_index) + window_index = torch.cat(window_indices) # compute reverse indices reverse_indices = self.invert_permutation(window_index) - cu_window_seqlens = torch.cat(cu_window_seqlens) + cu_window_seqlens = torch.cat(cu_window_seqlens_parts) cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens) - cu_seqlens = torch.cat(cu_seqlens) + cu_seqlens = torch.cat(cu_seqlens_parts) cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32) cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0) @@ -1088,6 +1091,7 @@ class Qwen2_5_VisionTransformer(nn.Module): seq_len = hidden_states.shape[0] if encoder_metadata is None: + assert grid_thw is not None encoder_metadata = self.prepare_encoder_metadata(grid_thw) rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"] @@ -1162,7 +1166,7 @@ class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder): ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -1216,6 +1220,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor): hf_data = hf_inputs.hf_data if videos := hf_data.get("videos"): + assert isinstance(videos, list) # HF expects video metadata as a separate argument. hf_data["videos"] = [video for video, _ in videos] hf_data["video_metadata"] = [ @@ -1336,17 +1341,22 @@ class Qwen2_5_VLForConditionalGeneration( tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0) for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset + data = mm_feature.data + assert data is not None if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid = data["image_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0 elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid = data["video_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() second_per_grid_ts = 1.0 - if mm_feature.data.get("second_per_grid_ts", None): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + if seconds := data.get("second_per_grid_ts"): + assert isinstance(seconds.data, torch.Tensor) + second_per_grid_ts = seconds.data.item() t_factor = second_per_grid_ts * tokens_per_second yield ( offset, @@ -1409,7 +1419,7 @@ class Qwen2_5_VLForConditionalGeneration( def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): super().__init__() config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config @@ -1458,7 +1468,7 @@ class Qwen2_5_VLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2_5_VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -1484,7 +1494,7 @@ class Qwen2_5_VLForConditionalGeneration( second_per_grid_ts=second_per_grid_ts, ) - if video_embeds is not None: + else: return Qwen2_5_VLVideoEmbeddingInputs( type="video_embeds", video_embeds=video_embeds, @@ -1549,8 +1559,7 @@ class Qwen2_5_VLForConditionalGeneration( ) emb = torch.cat([emb, positions], dim=1) image_embeds_out.append(emb) - image_embeds_split = image_embeds_out - return tuple(image_embeds_split) + return tuple(image_embeds_out) def _process_video_input( self, video_input: Qwen2_5_VLVideoInputs @@ -1614,6 +1623,7 @@ class Qwen2_5_VLForConditionalGeneration( tokens_per_second = self.config.vision_config.tokens_per_second video_embeds_out = [] + assert self.video_pruning_rate is not None for emb, size, video_second_per_grid_t in zip( video_embeds_split, grid_thw_list, second_per_grid_ts ): @@ -1702,7 +1712,9 @@ class Qwen2_5_VLForConditionalGeneration( return mm_embeddings_out, positions, mrope_positions_delta def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1808,6 +1820,7 @@ class Qwen2_5_VLForConditionalGeneration( def get_max_frames_per_video(self) -> int: mm_registry = MULTIMODAL_REGISTRY info = mm_registry.get_processing_info(self.model_config) + assert isinstance(info, Qwen2VLProcessingInfo) max_frames_per_video = info.get_num_frames_with_most_features( seq_len=self.model_config.max_model_len, mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, diff --git a/vllm/model_executor/models/qwen2_audio.py b/vllm/model_executor/models/qwen2_audio.py index dca856c45f..a305eaba89 100644 --- a/vllm/model_executor/models/qwen2_audio.py +++ b/vllm/model_executor/models/qwen2_audio.py @@ -286,11 +286,14 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing def get_replacement_qwen2_audio(item_idx: int): if "audio_num_tokens" in out_mm_data: - num_features = int(out_mm_data["audio_num_tokens"][item_idx]) + audio_num_tokens = out_mm_data["audio_num_tokens"] + assert isinstance(audio_num_tokens, torch.Tensor) + num_features = int(audio_num_tokens[item_idx]) elif audio_output_lengths: num_features = audio_output_lengths[item_idx] else: audio_embeds = out_mm_data["audio_embeds"][item_idx] + assert isinstance(audio_embeds, torch.Tensor) assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor" num_features = audio_embeds.shape[0] diff --git a/vllm/model_executor/models/qwen2_moe.py b/vllm/model_executor/models/qwen2_moe.py index 69d538f329..1c095d6f09 100644 --- a/vllm/model_executor/models/qwen2_moe.py +++ b/vllm/model_executor/models/qwen2_moe.py @@ -153,6 +153,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module): prefix=f"{prefix}.shared_expert_gate", ) + self.shared_expert: Qwen2MoeMLP | None if config.shared_expert_intermediate_size > 0: self.shared_expert = Qwen2MoeMLP( hidden_size=config.hidden_size, @@ -252,6 +253,12 @@ class Qwen2MoeAttention(nn.Module): rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = Attention( self.num_heads, self.head_dim, @@ -260,12 +267,7 @@ class Qwen2MoeAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) def forward( diff --git a/vllm/model_executor/models/qwen2_vl.py b/vllm/model_executor/models/qwen2_vl.py index 00d98dae7c..3dae3ffd2a 100644 --- a/vllm/model_executor/models/qwen2_vl.py +++ b/vllm/model_executor/models/qwen2_vl.py @@ -1211,17 +1211,23 @@ class Qwen2VLForConditionalGeneration( tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0) for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset + data = mm_feature.data + assert data is not None if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + image_grid_thw = data["image_grid_thw"] + assert isinstance(image_grid_thw.data, torch.Tensor) + t, h, w = image_grid_thw.data.tolist() assert t == 1, f"Image must have 1 frame, got {t}" yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0 elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + video_grid_thw = data["video_grid_thw"] + assert isinstance(video_grid_thw.data, torch.Tensor) + t, h, w = video_grid_thw.data.tolist() second_per_grid_ts = 1.0 - if mm_feature.data.get("second_per_grid_ts", None): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + second_per_grid_ts_field = data.get("second_per_grid_ts") + if second_per_grid_ts_field is not None: + assert isinstance(second_per_grid_ts_field.data, torch.Tensor) + second_per_grid_ts = second_per_grid_ts_field.data.item() t_factor = second_per_grid_ts * tokens_per_second yield ( offset, @@ -1285,7 +1291,7 @@ class Qwen2VLForConditionalGeneration( super().__init__() config: Qwen2VLConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.model_config = vllm_config.model_config self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.config = config @@ -1328,7 +1334,7 @@ class Qwen2VLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -1352,7 +1358,7 @@ class Qwen2VLForConditionalGeneration( video_grid_thw=video_grid_thw, ) - if video_embeds is not None: + else: return Qwen2VLVideoEmbeddingInputs( type="video_embeds", video_embeds=video_embeds, @@ -1411,7 +1417,7 @@ class Qwen2VLForConditionalGeneration( return video_embeds.split(sizes) def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - modalities = {} + modalities: dict[str, Qwen2VLImageInputs | Qwen2VLVideoInputs | None] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1481,6 +1487,7 @@ class Qwen2VLForConditionalGeneration( def get_max_frames_per_video(self) -> int: mm_registry = MULTIMODAL_REGISTRY info = mm_registry.get_processing_info(self.model_config) + assert isinstance(info, Qwen2VLProcessingInfo) max_frames_per_video = info.get_num_frames_with_most_features( seq_len=self.model_config.max_model_len, mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, diff --git a/vllm/model_executor/models/qwen3.py b/vllm/model_executor/models/qwen3.py index 79434bfac9..7a1d76234f 100644 --- a/vllm/model_executor/models/qwen3.py +++ b/vllm/model_executor/models/qwen3.py @@ -130,6 +130,12 @@ class Qwen3Attention(nn.Module): if attn_type == AttentionType.ENCODER_ONLY else Attention ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = attn_cls( self.num_heads, self.head_dim, @@ -140,12 +146,7 @@ class Qwen3Attention(nn.Module): per_layer_sliding_window=per_layer_sliding_window, prefix=f"{prefix}.attn", attn_type=attn_type, - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) diff --git a/vllm/model_executor/models/qwen3_5.py b/vllm/model_executor/models/qwen3_5.py index 9204d5305d..ac27296461 100644 --- a/vllm/model_executor/models/qwen3_5.py +++ b/vllm/model_executor/models/qwen3_5.py @@ -25,6 +25,7 @@ """Inference-only Qwen3.5 Series compatible with HuggingFace weights.""" from collections.abc import Iterable +from typing import ClassVar import torch from torch import nn @@ -54,6 +55,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import ( VocabParallelEmbedding, ) from vllm.multimodal import MULTIMODAL_REGISTRY +from vllm.multimodal.inputs import MultiModalFeatureSpec from vllm.sequence import IntermediateTensors from vllm.tokenizers.registry import cached_tokenizer_from_config from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig @@ -442,7 +444,7 @@ class Qwen3_5ForCausalLMBase( def get_mrope_input_positions( self, input_tokens: list[int], - mm_features: list[object], + mm_features: list[MultiModalFeatureSpec], ) -> tuple[torch.Tensor, int]: positions = torch.arange(len(input_tokens), dtype=torch.long) return positions.unsqueeze(0).expand(3, -1), 0 @@ -488,7 +490,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) nn.Module.__init__(self) config: Qwen3_5Config = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config @@ -648,6 +650,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid) class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts): + language_model: Qwen3_5ForCausalLM + num_local_physical_experts: int + def update_physical_experts_metadata( self, num_physical_experts: int, @@ -700,14 +705,14 @@ class Qwen3_5MoeForConditionalGeneration( Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts ): # For MoE LoRA weights loading - is_3d_moe_weight: bool = True + is_3d_moe_weight: ClassVar[bool] = True def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"): # protocols have not __init__ method, so we need to use nn.Module.__init__ nn.Module.__init__(self) config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config diff --git a/vllm/model_executor/models/qwen3_5_mtp.py b/vllm/model_executor/models/qwen3_5_mtp.py index 7fb699b183..421292e880 100644 --- a/vllm/model_executor/models/qwen3_5_mtp.py +++ b/vllm/model_executor/models/qwen3_5_mtp.py @@ -286,7 +286,7 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP): return inputs_embeds - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/qwen3_asr.py b/vllm/model_executor/models/qwen3_asr.py index cf8f37cebc..e8e291d8a5 100644 --- a/vllm/model_executor/models/qwen3_asr.py +++ b/vllm/model_executor/models/qwen3_asr.py @@ -274,7 +274,7 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser): class Qwen3ASRMultiModalProcessor( - Qwen3OmniMoeThinkerMultiModalProcessor, + Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo] ): def _get_mm_fields_config( self, @@ -316,6 +316,7 @@ class Qwen3ASRMultiModalProcessor( if num_features == 0: audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) + assert audio is not None raise ValueError( f"The audio {audio} (len={len(audio)}) is too short " "to be represented inside the model" diff --git a/vllm/model_executor/models/qwen3_asr_forced_aligner.py b/vllm/model_executor/models/qwen3_asr_forced_aligner.py index 2cbf4c1111..a7aad6628d 100644 --- a/vllm/model_executor/models/qwen3_asr_forced_aligner.py +++ b/vllm/model_executor/models/qwen3_asr_forced_aligner.py @@ -72,8 +72,9 @@ class Qwen3ASRForcedAlignerForTokenClassification( # Remove the unused generation head created by the base class; # the forced aligner uses a classifier head instead. - self.language_model.lm_head = None - self.language_model.logits_processor = None + language_model: nn.Module = self.language_model + language_model.lm_head = None + language_model.logits_processor = None self.classify_num = thinker_config.classify_num diff --git a/vllm/model_executor/models/qwen3_asr_realtime.py b/vllm/model_executor/models/qwen3_asr_realtime.py index 6eb762b628..ef6a0be73b 100644 --- a/vllm/model_executor/models/qwen3_asr_realtime.py +++ b/vllm/model_executor/models/qwen3_asr_realtime.py @@ -120,6 +120,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor): ) audio_data = audios[0] + assert audio_data is not None audio_feature_lengths = audio_data.get("audio_feature_lengths") if audio_feature_lengths is not None: if isinstance(audio_feature_lengths.data, torch.Tensor): diff --git a/vllm/model_executor/models/qwen3_dflash.py b/vllm/model_executor/models/qwen3_dflash.py index 0f02b98233..36a6c4c2c1 100644 --- a/vllm/model_executor/models/qwen3_dflash.py +++ b/vllm/model_executor/models/qwen3_dflash.py @@ -92,7 +92,9 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool: if override is not None: return bool(override) layer_types = getattr(config, "layer_types", None) - return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION + if not layer_types: + return False + return layer_types[layer_idx] == _SLIDING_ATTENTION def dflash_has_any_non_causal(config: Qwen3Config) -> bool: @@ -105,6 +107,7 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool: def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int: spec_config = vllm_config.speculative_config + assert spec_config is not None config = spec_config.draft_model_config.hf_config aux_layers = get_eagle3_aux_layers_from_config(spec_config) num_features_to_use = len(aux_layers) if aux_layers else config.num_hidden_layers @@ -418,7 +421,9 @@ class DFlashQwen3Model(nn.Module): prefix: str = "", ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size self.quant_config = get_draft_quant_config(vllm_config) _add_global_draft_layer_exclusions( @@ -759,7 +764,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.draft_model_config = vllm_config.speculative_config.draft_model_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.draft_model_config = speculative_config.draft_model_config self.config = self.draft_model_config.hf_config if getattr(self.config, "draft_vocab_size", None) is None: self.config.draft_vocab_size = getattr(self.config, "vocab_size", None) @@ -801,7 +808,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): ) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, @@ -896,7 +903,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM): model_weights["model.mask_embedding"] = mask_embedding self.model.has_separate_mask_embedding = True - orig_to_new_substr = {} + orig_to_new_substr: dict[str, None] = {} if not includes_draft_id_mapping: orig_to_new_substr["draft_id_to_target_id"] = None if not includes_embed_tokens: diff --git a/vllm/model_executor/models/qwen3_dspark.py b/vllm/model_executor/models/qwen3_dspark.py index 7318d225f8..156f55c373 100644 --- a/vllm/model_executor/models/qwen3_dspark.py +++ b/vllm/model_executor/models/qwen3_dspark.py @@ -233,7 +233,9 @@ class Qwen3DSparkModel(DFlashQwen3Model): class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: nn.Module.__init__(self) - self.draft_model_config = vllm_config.speculative_config.draft_model_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.draft_model_config = speculative_config.draft_model_config self.config = self.draft_model_config.hf_config if getattr(self.config, "draft_vocab_size", None) is None: self.config.draft_vocab_size = getattr(self.config, "vocab_size", None) diff --git a/vllm/model_executor/models/qwen3_eagle3.py b/vllm/model_executor/models/qwen3_eagle3.py index 745c77a977..124c1f82ca 100644 --- a/vllm/model_executor/models/qwen3_eagle3.py +++ b/vllm/model_executor/models/qwen3_eagle3.py @@ -101,7 +101,7 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer): hidden_states = self.hidden_norm(hidden_states) return hidden_states, residual - def forward( + def forward( # type: ignore[override] self, positions: torch.Tensor, embeds: torch.Tensor, @@ -159,7 +159,9 @@ class Qwen3Eagle3Model(nn.Module): prefix: str = "", ) -> None: super().__init__() - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config self.vocab_size = self.config.vocab_size # Get drafter's quantization config @@ -210,6 +212,7 @@ class Qwen3Eagle3Model(nn.Module): self.config, "target_hidden_size", self.config.hidden_size ) self.fc_input_size = target_hidden_size * num_aux_features + self.input_norm: RMSNorm | None if self.norm_before_fc: self.input_norm = RMSNorm( self.fc_input_size, @@ -282,7 +285,9 @@ class Qwen3Eagle3Model(nn.Module): class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): nn.Module.__init__(self) - self.config = vllm_config.speculative_config.draft_model_config.hf_config + speculative_config = vllm_config.speculative_config + assert speculative_config is not None + self.config = speculative_config.draft_model_config.hf_config # Ensure draft_vocab_size is set # default to the base vocab size when absent if getattr(self.config, "draft_vocab_size", None) is None: @@ -318,7 +323,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): else: self.draft_id_to_target_id = None - self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting + self.use_parallel_drafting = speculative_config.parallel_drafting if self.use_parallel_drafting: self.register_buffer( @@ -335,7 +340,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM): ) -> torch.Tensor: return self.model.embed_input_ids(input_ids) - def forward( + def forward( # type: ignore[override] self, input_ids: torch.Tensor, positions: torch.Tensor, diff --git a/vllm/model_executor/models/qwen3_moe.py b/vllm/model_executor/models/qwen3_moe.py index e2f7dcaec5..3f293550ac 100644 --- a/vllm/model_executor/models/qwen3_moe.py +++ b/vllm/model_executor/models/qwen3_moe.py @@ -177,6 +177,8 @@ class Qwen3MoeSparseMoeBlock(nn.Module): shared_expert_intermediate_size = getattr( config, "shared_expert_intermediate_size", 0 ) + self.shared_expert_gate: ReplicatedLinear | None + self.shared_expert: Qwen3MoeMLP | None if shared_expert_intermediate_size > 0: self.shared_expert_gate = ReplicatedLinear( config.hidden_size, @@ -302,6 +304,12 @@ class Qwen3MoeAttention(nn.Module): rope_parameters=rope_parameters, dual_chunk_attention_config=dual_chunk_attention_config, ) + attention_kwargs: dict[str, Any] = {} + if dual_chunk_attention_config: + attention_kwargs = { + "layer_idx": extract_layer_index(prefix), + "dual_chunk_attention_config": dual_chunk_attention_config, + } self.attn = Attention( self.num_heads, self.head_dim, @@ -310,12 +318,7 @@ class Qwen3MoeAttention(nn.Module): cache_config=cache_config, quant_config=quant_config, prefix=f"{prefix}.attn", - **{ - "layer_idx": extract_layer_index(prefix), - "dual_chunk_attention_config": dual_chunk_attention_config, - } - if dual_chunk_attention_config - else {}, + **attention_kwargs, ) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) diff --git a/vllm/model_executor/models/qwen3_next.py b/vllm/model_executor/models/qwen3_next.py index e779e608ea..e431c36f30 100644 --- a/vllm/model_executor/models/qwen3_next.py +++ b/vllm/model_executor/models/qwen3_next.py @@ -757,6 +757,9 @@ class Qwen3NextModel(nn.Module, EagleModelMixin): class QwenNextMixtureOfExperts(MixtureOfExperts): + model: Qwen3NextModel + num_local_physical_experts: int + def update_physical_experts_metadata( self, num_physical_experts: int, diff --git a/vllm/model_executor/models/qwen3_omni_moe_thinker.py b/vllm/model_executor/models/qwen3_omni_moe_thinker.py index bd48b554cd..f9338b1189 100755 --- a/vllm/model_executor/models/qwen3_omni_moe_thinker.py +++ b/vllm/model_executor/models/qwen3_omni_moe_thinker.py @@ -24,7 +24,7 @@ from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence from functools import partial -from typing import Any, cast +from typing import Any, ClassVar import numpy as np import torch @@ -41,12 +41,13 @@ from transformers.models.qwen3_omni_moe.processing_qwen3_omni_moe import ( Qwen3OmniMoeProcessor, ) from transformers.models.whisper import WhisperFeatureExtractor +from typing_extensions import TypeVar from vllm.compilation.decorators import support_torch_compile from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config.speech_to_text import SpeechToTextParams from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size -from vllm.inputs import PromptType +from vllm.inputs import PromptType, TextPrompt from vllm.logger import init_logger from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY from vllm.model_executor.layers.attention.mm_encoder_attention import ( @@ -73,6 +74,7 @@ from vllm.multimodal.inputs import ( ) from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems from vllm.multimodal.processing.processor import ( + BaseProcessingInfo, HFMultiModalInputs, MultiModalProcessingResult, MultiModalPromptUpdates, @@ -102,12 +104,15 @@ from .qwen2_5_omni_thinker import ( Qwen2_5OmniThinkerDummyInputsBuilder, Qwen2_5OmniThinkerMultiModalDataParser, Qwen2_5OmniThinkerMultiModalProcessor, + _get_feature_tensor, check_interleaved_audio_video, merge_interleaved_embeddings, ) from .qwen2_5_vl import ( Qwen2_5_VisionAttention, + Qwen2_5_VLImageInputs, Qwen2_5_VLProcessingInfo, + Qwen2_5_VLVideoInputs, ) from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel from .utils import ( @@ -963,7 +968,7 @@ class Qwen3Omni_VisionTransformer(nn.Module): def forward( self, x: torch.Tensor, - grid_thw: list[list[int]], + grid_thw: torch.Tensor, ) -> torch.Tensor: hidden_states = x.to(device=self.device, dtype=self.dtype) hidden_states = self.patch_embed(hidden_states) @@ -1201,7 +1206,7 @@ class Qwen3OmniMoeThinkerProcessingInfo( self, seq_len: int, mm_counts: Mapping[str, int] | None = None, - ) -> Mapping[str, int] | None: + ) -> Mapping[str, int]: mm_counts = mm_counts or {} requested_modalities = {m for m, c in mm_counts.items() if c > 0} mm_max_tokens: dict[str, int] = {} @@ -1231,11 +1236,15 @@ class Qwen3OmniMoeThinkerProcessingInfo( return mm_max_tokens -Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder +Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder[ + Qwen3OmniMoeThinkerProcessingInfo +] + +_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen3OmniMoeThinkerProcessingInfo) class Qwen3OmniMoeThinkerMultiModalProcessor( - Qwen2_5OmniThinkerMultiModalProcessor, + Qwen2_5OmniThinkerMultiModalProcessor[_I], ): def _get_hf_mm_inputs( self, @@ -1246,7 +1255,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( audios = hf_inputs.hf_data.get("audio") if audios: - feature_extractor = self.info.get_feature_extractor() + assert isinstance(audios, (list, tuple)) + feature_extractor = self.info.get_hf_processor().feature_extractor hop_length = feature_extractor.hop_length def pad_to_hop_length(x: np.ndarray) -> np.ndarray: @@ -1283,9 +1293,10 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( ): audios = mm_data.get("audio", []) if audios: - feature_extractor = self.info.get_feature_extractor( + assert isinstance(audios, (list, tuple)) + feature_extractor = self.info.get_hf_processor( **hf_processor_mm_kwargs - ) + ).feature_extractor hop_length = feature_extractor.hop_length audio_num_frames = [] @@ -1421,6 +1432,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( ) -> Sequence[PromptUpdate]: processor = self.info.get_hf_processor(**hf_processor_mm_kwargs) tokenizer = self.info.get_tokenizer() + assert isinstance(self.info, Qwen2_5_VLProcessingInfo) image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs) vocab = tokenizer.get_vocab() @@ -1460,6 +1472,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor( if num_features == 0: audios = mm_items.get_items("audio", AudioProcessorItems) audio = audios.get(item_idx) + assert audio is not None raise ValueError( f"The audio {audio} (len={len(audio)}) is too short " "to be represented inside the model" @@ -1607,8 +1620,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( Qwen3OmniMoeConditionalGenerationMixin, SupportsTranscription, ): - is_3d_moe_weight: bool = True - supports_tower_connector_lora: bool = True + is_3d_moe_weight: ClassVar[bool] = True + supports_tower_connector_lora: ClassVar[bool] = True hf_to_vllm_mapper = WeightsMapper( orig_to_new_prefix={ "thinker.lm_head.": "language_model.lm_head.", @@ -1769,7 +1782,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( self.deepstack_input_embeds_num_tokens = 0 def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, + Qwen2_5_VLImageInputs + | Qwen2_5_VLVideoInputs + | Qwen2_5OmniAudioFeatureInputs + | None, + ] = {} # Preserve the order of modalities if there are multiple of them # from the order of kwargs. @@ -1862,6 +1881,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( if self.visual.deepstack_visual_indexes is not None and any( has_vision_embeddings ): + multimodal_embeddings = list(multimodal_embeddings) multiscale_len = len(self.visual.deepstack_visual_indexes) multimodal_embeddings_multiscale = [] @@ -2006,8 +2026,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( f for f in mm_features if f.modality == "video" + and f.data is not None and f.data.get("use_audio_in_video") - and f.data["use_audio_in_video"].data.item() + and _get_feature_tensor(f, "use_audio_in_video").item() ] audios = [f for f in mm_features if f.modality == "audio"] @@ -2015,7 +2036,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( paired_audio_offsets: set[int] = set() for i, video_f in enumerate(videos_with_audio): if i < len(audios): - audio_len = audios[i].data["audio_feature_lengths"].data.item() + audio_len = int( + _get_feature_tensor(audios[i], "audio_feature_lengths").item() + ) mapping[video_f.mm_position.offset] = audio_len paired_audio_offsets.add(audios[i].mm_position.offset) return mapping, paired_audio_offsets @@ -2043,9 +2066,10 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( for mm_feature in sorted_features: offset = mm_feature.mm_position.offset modality = mm_feature.modality + assert mm_feature.data is not None if modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist() yield ( offset, "image", @@ -2057,15 +2081,15 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( }, ) elif modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist() second_per_grid_ts = 2.0 if mm_feature.data.get("second_per_grid_ts"): - second_per_grid_ts = mm_feature.data[ - "second_per_grid_ts" - ].data.item() + second_per_grid_ts = _get_feature_tensor( + mm_feature, "second_per_grid_ts" + ).item() use_audio_in_video = bool( mm_feature.data.get("use_audio_in_video") - and mm_feature.data["use_audio_in_video"].data.item() + and _get_feature_tensor(mm_feature, "use_audio_in_video").item() ) yield ( @@ -2082,7 +2106,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ) elif modality == "audio": if offset not in paired_audio_offsets: - audio_len = mm_feature.data["audio_feature_lengths"].data.item() + audio_len = _get_feature_tensor( + mm_feature, "audio_feature_lengths" + ).item() yield offset, "audio", {"audio_feature_length": audio_len} def _compute_interleaved_positions( @@ -2170,8 +2196,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( to_language = "en" # Get full language names from supported_languages mapping - full_lang_name = cls.supported_languages.get(language, "") - full_lang_name_to = cls.supported_languages.get(to_language, "") + full_lang_name = ( + cls.supported_languages.get(language, "") if language is not None else "" + ) + full_lang_name_to = ( + cls.supported_languages.get(to_language, "") + if to_language is not None + else "" + ) if task_type == "transcribe" and full_lang_name: instruction += f" into {full_lang_name}" @@ -2201,8 +2233,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration( ) audio_data = (audio, stt_config.sample_rate) - prompts_dict = {"multi_modal_data": {"audio": audio_data}, "prompt": prompt} - return cast(PromptType, prompts_dict) + prompts_dict: TextPrompt = { + "multi_modal_data": {"audio": audio_data}, + "prompt": prompt, + } + return prompts_dict def get_mrope_input_positions( self, diff --git a/vllm/model_executor/models/qwen3_vl.py b/vllm/model_executor/models/qwen3_vl.py index 619f2aaad4..c9aa0d2066 100644 --- a/vllm/model_executor/models/qwen3_vl.py +++ b/vllm/model_executor/models/qwen3_vl.py @@ -582,7 +582,9 @@ class Qwen3_VisionTransformer(nn.Module): else [] ) self.num_grid_per_side = int(self.num_position_embeddings**0.5) - self._rot_pos_ids_cache = LRUCache(capacity=1024) + self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = ( + LRUCache(capacity=1024) + ) use_data_parallel = is_vit_use_data_parallel() self.tp_size = ( @@ -705,7 +707,7 @@ class Qwen3_VisionTransformer(nn.Module): self._rot_pos_ids_cache[cache_key] = result return result - def rot_pos_emb(self, grid_thw: list[list[int]]): + def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]): max_grid_size = max(max(h, w) for _, h, w in grid_thw) pos_ids = [ self.rot_pos_ids(h, w, self.spatial_merge_size) @@ -731,7 +733,9 @@ class Qwen3_VisionTransformer(nn.Module): return cos_combined, sin_combined - def fast_pos_embed_interpolate(self, grid_thw: list[list[int]]) -> torch.Tensor: + def fast_pos_embed_interpolate( + self, grid_thw: Sequence[Sequence[int]] + ) -> torch.Tensor: interpolate_fn = ( triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native ) @@ -752,7 +756,7 @@ class Qwen3_VisionTransformer(nn.Module): def prepare_encoder_metadata( self, - grid_thw_list: list[list[int]], + grid_thw_list: Sequence[Sequence[int]], *, max_batch_size: int | None = None, max_frames_per_batch: int | None = None, @@ -1005,9 +1009,10 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo): self, seq_len: int, mm_counts: Mapping[str, int], + max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES, ) -> int: return super().get_num_frames_with_most_features( - seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES + seq_len, mm_counts, max_frames_per_video=max_frames_per_video ) def get_max_video_tokens( @@ -1253,7 +1258,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]): ) videos = [v.copy() for v in videos] - video_items = [] + video_items: list[VideoItem] = [] for video in videos: video_num_frames = video.shape[0] video_metadata = { @@ -1339,6 +1344,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # are processed into several image patches video_input_ids_lst: list[list[int]] = [] if videos := hf_data.pop("videos", []): + assert isinstance(videos, list) video_grid_thw_lst = [] pixel_values_videos_lst = [] timestamps_per_video = [] @@ -1361,7 +1367,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) # NOTE: a copy of is created to update do_sample_frames, # otherwise mm_hash for the object will be incorrect. - video_mm_kwargs = dict(**hf_kwargs) + video_mm_kwargs: dict[str, Any] = dict(hf_kwargs) merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video") if merged.keys() & {"size", "min_pixels", "max_pixels"}: video_size = dict(self.info.get_video_processor().size) @@ -1551,6 +1557,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) sampled_fps = sampled_fps[item_idx] timestamps = out_item["timestamps"].data + assert isinstance(timestamps, (list, torch.Tensor)) assert len(timestamps) == grid_thw[0], ( f"The timestamps length({len(timestamps)}) should be equal " f"video length ({grid_thw[0]})." @@ -1617,7 +1624,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo]) def get_video_repl( *, tokens_per_frame: list[int], - timestamps: list[float | int] | torch.Tensor, + timestamps: Sequence[float] | torch.Tensor, tokenizer: TokenizerLike, vision_start_token_id: int, vision_end_token_id: int, @@ -1842,7 +1849,7 @@ class Qwen3VLForConditionalGeneration( super().__init__() config: Qwen3VLConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config @@ -2006,6 +2013,7 @@ class Qwen3VLForConditionalGeneration( def get_max_frames_per_video(self) -> int: mm_registry = MULTIMODAL_REGISTRY info = mm_registry.get_processing_info(self.model_config) + assert isinstance(info, Qwen3VLProcessingInfo) max_frames_per_video = info.get_num_frames_with_most_features( seq_len=self.model_config.max_model_len, mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")}, @@ -2254,7 +2262,7 @@ class Qwen3VLForConditionalGeneration( image_grid_thw=image_grid_thw, ) - if image_embeds is not None: + else: return Qwen2_5_VLImageEmbeddingInputs( type="image_embeds", image_embeds=image_embeds, @@ -2282,7 +2290,7 @@ class Qwen3VLForConditionalGeneration( timestamps=timestamps, ) - if video_embeds is not None: + else: return Qwen2_5_VLVideoEmbeddingInputs( type="video_embeds", video_embeds=video_embeds, @@ -2405,6 +2413,7 @@ class Qwen3VLForConditionalGeneration( # Apply EVS to each video. video_embeds_out = [] + assert video_input.timestamps is not None for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)): # Compute positions. timestamps = video_input.timestamps[video_idx] @@ -2412,6 +2421,7 @@ class Qwen3VLForConditionalGeneration( t, h, w = size if self.is_multimodal_pruning_enabled: + assert self.video_pruning_rate is not None # Compute the retention mask for each video (EVS or VidCom2). if self.video_pruning_method == "vidcom2": mask_fn = vidcom2_compute_retention_mask @@ -2583,7 +2593,7 @@ class Qwen3VLForConditionalGeneration( { "video_grid_thw": MultiModalFieldElem( data=torch.tensor(video_grid_thw), - field=None, # HACK. + field=MultiModalFieldConfig.batched("video").field, ), } ), @@ -2611,7 +2621,9 @@ class Qwen3VLForConditionalGeneration( return expanded_positions def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: - mm_input_by_modality = {} + mm_input_by_modality: dict[ + str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None + ] = {} for input_key in kwargs: if ( input_key in ("pixel_values", "image_embeds") @@ -2659,14 +2671,20 @@ class Qwen3VLForConditionalGeneration( """ for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset): offset = mm_feature.mm_position.offset + data = mm_feature.data + assert data is not None if mm_feature.modality == "image": - t, h, w = mm_feature.data["image_grid_thw"].data.tolist() + grid = data["image_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() assert t == 1, f"Image must have 1 frame, got {t}" llm_grid_h = h // spatial_merge_size llm_grid_w = w // spatial_merge_size yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w elif mm_feature.modality == "video": - t, h, w = mm_feature.data["video_grid_thw"].data.tolist() + grid = data["video_grid_thw"].data + assert isinstance(grid, torch.Tensor) + t, h, w = grid.tolist() llm_grid_h = h // spatial_merge_size llm_grid_w = w // spatial_merge_size @@ -2721,7 +2739,7 @@ class Qwen3VLForConditionalGeneration( mm_features: list[MultiModalFeatureSpec], config: Qwen3VLConfig, ): - llm_pos_ids_list = [] + llm_pos_ids_list: list[np.ndarray] = [] st = 0 for ( offset, diff --git a/vllm/model_executor/models/qwen3_vl_moe.py b/vllm/model_executor/models/qwen3_vl_moe.py index 193298a186..a345f577b1 100644 --- a/vllm/model_executor/models/qwen3_vl_moe.py +++ b/vllm/model_executor/models/qwen3_vl_moe.py @@ -25,6 +25,7 @@ """Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights.""" from itertools import islice +from typing import ClassVar import torch from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import ( @@ -176,6 +177,9 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM): class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts): + language_model: Qwen3MoeLLMForCausalLM + num_local_physical_experts: int + def update_physical_experts_metadata( self, num_physical_experts: int, @@ -223,7 +227,7 @@ class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts): class Qwen3VLMoeForConditionalGeneration( Qwen3VLForConditionalGeneration, Qwen3VLMoeMixtureOfExperts ): - is_3d_moe_weight: bool = True + is_3d_moe_weight: ClassVar[bool] = True packed_modules_mapping = { "qkv_proj": [ "q_proj", @@ -236,7 +240,7 @@ class Qwen3VLMoeForConditionalGeneration( super(Qwen3VLForConditionalGeneration, self).__init__() config: Qwen3VLMoeConfig = vllm_config.model_config.hf_config quant_config = vllm_config.quant_config - multimodal_config = vllm_config.model_config.multimodal_config + multimodal_config = vllm_config.model_config.get_multimodal_config() self.config = config self.model_config = vllm_config.model_config @@ -293,8 +297,8 @@ class Qwen3VLMoeForConditionalGeneration( ) # Whether to include the gate_up_proj mapping is determined by - # the language model. - self.packed_modules_mapping = ( + # the language model. Keep this override local to the instance. + self.packed_modules_mapping = ( # type: ignore[misc] self.packed_modules_mapping | self.language_model.packed_modules_mapping )