[Mypy] Fix mypy typing for Qwen and Qianfan models (#58046)

Signed-off-by: Ashraf Bhuiyan <mbhuiyan@redhat.com>
This commit is contained in:
Ashraf Bhuiyan
2026-09-26 14:31:51 +00:00
committed by GitHub
parent dfab504333
commit 5113cf9d16
23 changed files with 302 additions and 157 deletions
-1
View File
@@ -113,7 +113,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [
r"vllm/model_executor/models/[kK]",
r"vllm/model_executor/models/[qQ]",
r"vllm/model_executor/models/[uU]",
r"vllm/model_executor/models/[vV]",
r"vllm/model_executor/models/[wW]",
+1 -1
View File
@@ -1262,7 +1262,7 @@ class SupportsRealtime(Protocol):
Override in subclasses based on the model's expected output length."""
@classmethod
async def buffer_realtime_audio(
def buffer_realtime_audio(
cls,
audio_stream: AsyncGenerator[np.ndarray, None],
input_stream: asyncio.Queue[list[int]],
+8 -8
View File
@@ -36,14 +36,14 @@ class QianfanOCRProcessingInfo(BaseInternVLProcessingInfo):
config = self.get_hf_config()
vision_config = config.vision_config
kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
kwargs.setdefault("image_size", vision_config.image_size)
kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
kwargs.setdefault("use_thumbnail", config.use_thumbnail)
merged_kwargs = self.ctx.get_merged_mm_kwargs(kwargs)
merged_kwargs.setdefault("image_size", vision_config.image_size)
merged_kwargs.setdefault("min_dynamic_patch", config.min_dynamic_patch)
merged_kwargs.setdefault("max_dynamic_patch", config.max_dynamic_patch)
merged_kwargs.setdefault("dynamic_image_size", config.dynamic_image_size)
merged_kwargs.setdefault("use_thumbnail", config.use_thumbnail)
image_processor = InternVLImageProcessor(**kwargs)
image_processor = InternVLImageProcessor(**merged_kwargs)
image_size = image_processor.image_size
patch_size = vision_config.patch_size
downsample_ratio = config.downsample_ratio
@@ -73,7 +73,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):
"""
def _patch_quant_config(
self, config: PreTrainedConfig, quant_config: QuantizationConfig
self, config: PreTrainedConfig, quant_config: QuantizationConfig | None
) -> None:
super()._patch_quant_config(config, quant_config)
# ignore vit layers to preserve model performance
+7 -6
View File
@@ -187,6 +187,12 @@ class Qwen2Attention(nn.Module):
if attn_type == AttentionType.ENCODER_ONLY
else Attention
)
attention_kwargs: dict[str, Any] = {}
if dual_chunk_attention_config:
attention_kwargs = {
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
self.attn = attn_cls(
self.num_heads,
self.head_dim,
@@ -196,12 +202,7 @@ class Qwen2Attention(nn.Module):
quant_config=quant_config,
attn_type=attn_type,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attention_kwargs,
)
def forward(
@@ -42,6 +42,7 @@ from transformers.models.qwen2_5_omni.processing_qwen2_5_omni import (
Qwen2_5OmniProcessor,
)
from transformers.models.whisper import WhisperFeatureExtractor
from typing_extensions import TypeVar
from vllm.config import VllmConfig
from vllm.config.multimodal import MultiModalDummyOptions
@@ -63,7 +64,10 @@ from vllm.model_executor.models.qwen2_audio import (
Qwen2AudioProcessingInfo,
_get_feat_extract_output_lengths,
)
from vllm.model_executor.models.qwen2_vl import Qwen2VLMultiModalDataParser
from vllm.model_executor.models.qwen2_vl import (
Qwen2VLMultiModalDataParser,
Qwen2VLProcessingInfo,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import (
ImageItem,
@@ -79,6 +83,7 @@ from vllm.multimodal.parse import (
)
from vllm.multimodal.processing import (
BaseDummyInputsBuilder,
BaseProcessingInfo,
)
from vllm.multimodal.processing.processor import (
BaseMultiModalProcessor,
@@ -118,6 +123,14 @@ except (ImportError, ModuleNotFoundError):
logger = init_logger(__name__)
def _get_feature_tensor(feature: MultiModalFeatureSpec, key: str) -> torch.Tensor:
data = feature.data
assert data is not None
tensor = data[key].data
assert isinstance(tensor, torch.Tensor)
return tensor
def unpad_and_flat_audio_features(
input_audio_features: torch.Tensor,
audio_feature_lengths: torch.Tensor,
@@ -347,7 +360,7 @@ class Qwen2_5OmniThinkerMultiModalDataParser(Qwen2VLMultiModalDataParser):
def _parse_audio_data(
self,
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any]:
) -> ModalityDataItems[Any, Any] | None:
if isinstance(data, dict):
required, optional = self.embedding_field_sets("audio")
data = dict(data)
@@ -406,7 +419,7 @@ class Qwen2_5OmniThinkerProcessingInfo(
self,
seq_len: int,
mm_counts: Mapping[str, int] | None = None,
) -> Mapping[str, int] | None:
) -> Mapping[str, int]:
mm_counts = mm_counts or {}
requested_modalities = {m for m, c in mm_counts.items() if c > 0}
mm_max_tokens: dict[str, int] = {}
@@ -436,9 +449,13 @@ class Qwen2_5OmniThinkerProcessingInfo(
return mm_max_tokens
class Qwen2_5OmniThinkerDummyInputsBuilder(
BaseDummyInputsBuilder[Qwen2_5OmniThinkerProcessingInfo]
):
_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo)
_VI = TypeVar(
"_VI", bound=Qwen2VLProcessingInfo, default=Qwen2_5OmniThinkerProcessingInfo
)
class Qwen2_5OmniThinkerDummyInputsBuilder(BaseDummyInputsBuilder[_VI]):
def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str:
num_audios = mm_counts.get("audio", 0)
num_images = mm_counts.get("image", 0)
@@ -462,7 +479,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
mm_counts: Mapping[str, int],
mm_options: MultiModalDummyOptions,
) -> MultiModalDataDict:
feature_extractor = self.info.get_feature_extractor()
feature_extractor = self.info.get_hf_processor().feature_extractor
target_audio_length = (
min(
@@ -501,9 +518,7 @@ class Qwen2_5OmniThinkerDummyInputsBuilder(
return mm_data
class Qwen2_5OmniThinkerMultiModalProcessor(
BaseMultiModalProcessor[Qwen2_5OmniThinkerProcessingInfo]
):
class Qwen2_5OmniThinkerMultiModalProcessor(BaseMultiModalProcessor[_I]):
def _get_hf_mm_text(self, mm_counts: Mapping[str, int]) -> str:
return self.dummy_inputs.get_dummy_text(mm_counts)
@@ -651,6 +666,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
for item in mm_kwargs["video"]:
if item and item.get("use_audio_in_video"):
use_audio_in_video_tensor = item["use_audio_in_video"].data
assert isinstance(use_audio_in_video_tensor, torch.Tensor)
if use_audio_in_video_tensor.numel() > 0:
use_audio_in_video = bool(use_audio_in_video_tensor.item())
break
@@ -749,6 +765,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
) -> Sequence[PromptUpdate]:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
assert isinstance(self.info, Qwen2VLProcessingInfo)
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
vocab = tokenizer.get_vocab()
@@ -786,6 +803,7 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
if num_features == 0:
audios = mm_items.get_items("audio", AudioProcessorItems)
audio = audios.get(item_idx)
assert audio is not None
raise ValueError(
f"The audio {audio} (len={len(audio)}) is too short "
"to be represented inside the model"
@@ -887,6 +905,10 @@ class Qwen2_5OmniThinkerMultiModalProcessor(
class Qwen2_5OmniConditionalGenerationMixin:
audio_tower: nn.Module
visual: nn.Module
vllm_config: VllmConfig
def _parse_and_validate_audio_input(
self, **kwargs: object
) -> Qwen2_5OmniAudioFeatureInputs | None:
@@ -916,7 +938,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
def _parse_and_validate_image_input(
self,
**kwargs: dict[str, Any],
**kwargs: object,
) -> Qwen2_5_VLImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -932,7 +954,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
else:
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
@@ -941,7 +963,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
def _parse_and_validate_video_input(
self,
**kwargs: dict[str, Any],
**kwargs: object,
) -> Qwen2_5_VLVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
@@ -957,7 +979,7 @@ class Qwen2_5OmniConditionalGenerationMixin:
video_grid_thw=video_grid_thw,
)
if video_embeds is not None:
else:
if not isinstance(video_embeds, torch.Tensor):
raise ValueError(
"Incorrect type of video embeddings. "
@@ -1130,7 +1152,13 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
mm_input_by_modality: dict[
str,
Qwen2_5_VLImageInputs
| Qwen2_5_VLVideoInputs
| Qwen2_5OmniAudioFeatureInputs
| None,
] = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -1175,8 +1203,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
f
for f in mm_features
if f.modality == "video"
and f.data is not None
and f.data.get("use_audio_in_video")
and f.data["use_audio_in_video"].data.item()
and _get_feature_tensor(f, "use_audio_in_video").item()
]
audios = [f for f in mm_features if f.modality == "audio"]
@@ -1185,7 +1214,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
paired_audio_offsets: set[int] = set()
for i, video_f in enumerate(videos_with_audio):
if i < len(audios):
audio_len = audios[i].data["audio_feature_lengths"].data.item()
audio_len = int(
_get_feature_tensor(audios[i], "audio_feature_lengths").item()
)
mapping[video_f.mm_position.offset] = audio_len
paired_audio_offsets.add(audios[i].mm_position.offset)
return mapping, paired_audio_offsets
@@ -1220,9 +1251,10 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
for mm_feature in sorted_features:
offset = mm_feature.mm_position.offset
modality = mm_feature.modality
assert mm_feature.data is not None
if modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
yield (
offset,
"image",
@@ -1234,16 +1266,16 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
},
)
elif modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
second_per_grid_ts = 1.0
if mm_feature.data.get("second_per_grid_ts"):
second_per_grid_ts = mm_feature.data[
"second_per_grid_ts"
].data.item()
second_per_grid_ts = _get_feature_tensor(
mm_feature, "second_per_grid_ts"
).item()
use_audio_in_video = False
if mm_feature.data.get("use_audio_in_video"):
use_audio_in_video = bool(
mm_feature.data["use_audio_in_video"].data.item()
_get_feature_tensor(mm_feature, "use_audio_in_video").item()
)
yield (
@@ -1261,7 +1293,9 @@ class Qwen2_5OmniThinkerForConditionalGeneration(
elif modality == "audio":
# Skip audio that's paired with video (handled in video case)
if offset not in paired_audio_offsets:
audio_len = mm_feature.data["audio_feature_lengths"].data.item()
audio_len = _get_feature_tensor(
mm_feature, "audio_feature_lengths"
).item()
yield offset, "audio", {"audio_feature_length": audio_len}
def _compute_interleaved_positions(
+37 -24
View File
@@ -654,7 +654,10 @@ class Qwen2_5_VisionTransformer(nn.Module):
self.spatial_merge_size = vision_config.spatial_merge_size
self.fullatt_block_indexes = vision_config.fullatt_block_indexes
self.spatial_merge_unit = self.spatial_merge_size**2
self._rope_by_thw_cache = LRUCache(capacity=1024)
self._rope_by_thw_cache: LRUCache[
tuple[int, int, int],
tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor],
] = LRUCache(capacity=1024)
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
1
@@ -885,7 +888,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
def prepare_encoder_metadata(
self,
grid_thw: list[list[int]],
grid_thw: Sequence[Sequence[int]],
*,
max_batch_size: int | None = None,
max_frames_per_batch: int | None = None,
@@ -927,9 +930,9 @@ class Qwen2_5_VisionTransformer(nn.Module):
# patchify
rotary_pos_emb_cos = []
rotary_pos_emb_sin = []
window_index: list = []
cu_window_seqlens: list = [torch.tensor([0], dtype=torch.int32)]
cu_seqlens: list = []
window_indices: list[torch.Tensor] = []
cu_window_seqlens_parts = [torch.tensor([0], dtype=torch.int32)]
cu_seqlens_parts: list[torch.Tensor] = []
window_index_id = 0
cu_window_seqlens_last = 0
@@ -946,26 +949,26 @@ class Qwen2_5_VisionTransformer(nn.Module):
cu_seqlens_thw,
) = self.get_rope_by_thw(t, h, w)
window_index.append(window_index_thw + window_index_id)
window_indices.append(window_index_thw + window_index_id)
window_index_id += t * llm_h * llm_w
cu_seqlens_window_thw = cu_seqlens_window_thw + cu_window_seqlens_last
cu_window_seqlens_last = cu_seqlens_window_thw[-1]
cu_window_seqlens.append(cu_seqlens_window_thw)
cu_window_seqlens_parts.append(cu_seqlens_window_thw)
rotary_pos_emb_cos.append(cos_thw)
rotary_pos_emb_sin.append(sin_thw)
cu_seqlens.append(cu_seqlens_thw)
cu_seqlens_parts.append(cu_seqlens_thw)
rotary_pos_emb_cos = torch.cat(rotary_pos_emb_cos)
rotary_pos_emb_sin = torch.cat(rotary_pos_emb_sin)
window_index = torch.cat(window_index)
window_index = torch.cat(window_indices)
# compute reverse indices
reverse_indices = self.invert_permutation(window_index)
cu_window_seqlens = torch.cat(cu_window_seqlens)
cu_window_seqlens = torch.cat(cu_window_seqlens_parts)
cu_window_seqlens = torch.unique_consecutive(cu_window_seqlens)
cu_seqlens = torch.cat(cu_seqlens)
cu_seqlens = torch.cat(cu_seqlens_parts)
cu_seqlens = torch.cumsum(cu_seqlens, dim=0, dtype=torch.int32)
cu_seqlens = F.pad(cu_seqlens, (1, 0), "constant", 0)
@@ -1088,6 +1091,7 @@ class Qwen2_5_VisionTransformer(nn.Module):
seq_len = hidden_states.shape[0]
if encoder_metadata is None:
assert grid_thw is not None
encoder_metadata = self.prepare_encoder_metadata(grid_thw)
rotary_pos_emb_cos = encoder_metadata["rotary_pos_emb_cos"]
@@ -1162,7 +1166,7 @@ class Qwen2_5_VLDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
)
videos = [v.copy() for v in videos]
video_items = []
video_items: list[VideoItem] = []
for video in videos:
video_num_frames = video.shape[0]
video_metadata = {
@@ -1216,6 +1220,7 @@ class Qwen2_5_VLMultiModalProcessor(Qwen2VLMultiModalProcessor):
hf_data = hf_inputs.hf_data
if videos := hf_data.get("videos"):
assert isinstance(videos, list)
# HF expects video metadata as a separate argument.
hf_data["videos"] = [video for video, _ in videos]
hf_data["video_metadata"] = [
@@ -1336,17 +1341,22 @@ class Qwen2_5_VLForConditionalGeneration(
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset
data = mm_feature.data
assert data is not None
if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
grid = data["image_grid_thw"].data
assert isinstance(grid, torch.Tensor)
t, h, w = grid.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
elif mm_feature.modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
grid = data["video_grid_thw"].data
assert isinstance(grid, torch.Tensor)
t, h, w = grid.tolist()
second_per_grid_ts = 1.0
if mm_feature.data.get("second_per_grid_ts", None):
second_per_grid_ts = mm_feature.data[
"second_per_grid_ts"
].data.item()
if seconds := data.get("second_per_grid_ts"):
assert isinstance(seconds.data, torch.Tensor)
second_per_grid_ts = seconds.data.item()
t_factor = second_per_grid_ts * tokens_per_second
yield (
offset,
@@ -1409,7 +1419,7 @@ class Qwen2_5_VLForConditionalGeneration(
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config: Qwen2_5_VLConfig = vllm_config.model_config.hf_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.config = config
@@ -1458,7 +1468,7 @@ class Qwen2_5_VLForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
else:
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
@@ -1484,7 +1494,7 @@ class Qwen2_5_VLForConditionalGeneration(
second_per_grid_ts=second_per_grid_ts,
)
if video_embeds is not None:
else:
return Qwen2_5_VLVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
@@ -1549,8 +1559,7 @@ class Qwen2_5_VLForConditionalGeneration(
)
emb = torch.cat([emb, positions], dim=1)
image_embeds_out.append(emb)
image_embeds_split = image_embeds_out
return tuple(image_embeds_split)
return tuple(image_embeds_out)
def _process_video_input(
self, video_input: Qwen2_5_VLVideoInputs
@@ -1614,6 +1623,7 @@ class Qwen2_5_VLForConditionalGeneration(
tokens_per_second = self.config.vision_config.tokens_per_second
video_embeds_out = []
assert self.video_pruning_rate is not None
for emb, size, video_second_per_grid_t in zip(
video_embeds_split, grid_thw_list, second_per_grid_ts
):
@@ -1702,7 +1712,9 @@ class Qwen2_5_VLForConditionalGeneration(
return mm_embeddings_out, positions, mrope_positions_delta
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
mm_input_by_modality: dict[
str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
] = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -1808,6 +1820,7 @@ class Qwen2_5_VLForConditionalGeneration(
def get_max_frames_per_video(self) -> int:
mm_registry = MULTIMODAL_REGISTRY
info = mm_registry.get_processing_info(self.model_config)
assert isinstance(info, Qwen2VLProcessingInfo)
max_frames_per_video = info.get_num_frames_with_most_features(
seq_len=self.model_config.max_model_len,
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
+4 -1
View File
@@ -286,11 +286,14 @@ class Qwen2AudioMultiModalProcessor(BaseMultiModalProcessor[Qwen2AudioProcessing
def get_replacement_qwen2_audio(item_idx: int):
if "audio_num_tokens" in out_mm_data:
num_features = int(out_mm_data["audio_num_tokens"][item_idx])
audio_num_tokens = out_mm_data["audio_num_tokens"]
assert isinstance(audio_num_tokens, torch.Tensor)
num_features = int(audio_num_tokens[item_idx])
elif audio_output_lengths:
num_features = audio_output_lengths[item_idx]
else:
audio_embeds = out_mm_data["audio_embeds"][item_idx]
assert isinstance(audio_embeds, torch.Tensor)
assert len(audio_embeds.shape) == 2, "audio_embeds must be a 2D tensor"
num_features = audio_embeds.shape[0]
+8 -6
View File
@@ -153,6 +153,7 @@ class Qwen2MoeSparseMoeBlock(nn.Module):
prefix=f"{prefix}.shared_expert_gate",
)
self.shared_expert: Qwen2MoeMLP | None
if config.shared_expert_intermediate_size > 0:
self.shared_expert = Qwen2MoeMLP(
hidden_size=config.hidden_size,
@@ -252,6 +253,12 @@ class Qwen2MoeAttention(nn.Module):
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
attention_kwargs: dict[str, Any] = {}
if dual_chunk_attention_config:
attention_kwargs = {
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -260,12 +267,7 @@ class Qwen2MoeAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attention_kwargs,
)
def forward(
+17 -10
View File
@@ -1211,17 +1211,23 @@ class Qwen2VLForConditionalGeneration(
tokens_per_second = getattr(self.config.vision_config, "tokens_per_second", 1.0)
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset
data = mm_feature.data
assert data is not None
if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
image_grid_thw = data["image_grid_thw"]
assert isinstance(image_grid_thw.data, torch.Tensor)
t, h, w = image_grid_thw.data.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
yield offset, 1, h // spatial_merge_size, w // spatial_merge_size, 1.0
elif mm_feature.modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
video_grid_thw = data["video_grid_thw"]
assert isinstance(video_grid_thw.data, torch.Tensor)
t, h, w = video_grid_thw.data.tolist()
second_per_grid_ts = 1.0
if mm_feature.data.get("second_per_grid_ts", None):
second_per_grid_ts = mm_feature.data[
"second_per_grid_ts"
].data.item()
second_per_grid_ts_field = data.get("second_per_grid_ts")
if second_per_grid_ts_field is not None:
assert isinstance(second_per_grid_ts_field.data, torch.Tensor)
second_per_grid_ts = second_per_grid_ts_field.data.item()
t_factor = second_per_grid_ts * tokens_per_second
yield (
offset,
@@ -1285,7 +1291,7 @@ class Qwen2VLForConditionalGeneration(
super().__init__()
config: Qwen2VLConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.model_config = vllm_config.model_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.config = config
@@ -1328,7 +1334,7 @@ class Qwen2VLForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
else:
return Qwen2VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
@@ -1352,7 +1358,7 @@ class Qwen2VLForConditionalGeneration(
video_grid_thw=video_grid_thw,
)
if video_embeds is not None:
else:
return Qwen2VLVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
@@ -1411,7 +1417,7 @@ class Qwen2VLForConditionalGeneration(
return video_embeds.split(sizes)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
modalities = {}
modalities: dict[str, Qwen2VLImageInputs | Qwen2VLVideoInputs | None] = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -1481,6 +1487,7 @@ class Qwen2VLForConditionalGeneration(
def get_max_frames_per_video(self) -> int:
mm_registry = MULTIMODAL_REGISTRY
info = mm_registry.get_processing_info(self.model_config)
assert isinstance(info, Qwen2VLProcessingInfo)
max_frames_per_video = info.get_num_frames_with_most_features(
seq_len=self.model_config.max_model_len,
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
+7 -6
View File
@@ -130,6 +130,12 @@ class Qwen3Attention(nn.Module):
if attn_type == AttentionType.ENCODER_ONLY
else Attention
)
attention_kwargs: dict[str, Any] = {}
if dual_chunk_attention_config:
attention_kwargs = {
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
self.attn = attn_cls(
self.num_heads,
self.head_dim,
@@ -140,12 +146,7 @@ class Qwen3Attention(nn.Module):
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{prefix}.attn",
attn_type=attn_type,
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attention_kwargs,
)
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
+9 -4
View File
@@ -25,6 +25,7 @@
"""Inference-only Qwen3.5 Series compatible with HuggingFace weights."""
from collections.abc import Iterable
from typing import ClassVar
import torch
from torch import nn
@@ -54,6 +55,7 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
VocabParallelEmbedding,
)
from vllm.multimodal import MULTIMODAL_REGISTRY
from vllm.multimodal.inputs import MultiModalFeatureSpec
from vllm.sequence import IntermediateTensors
from vllm.tokenizers.registry import cached_tokenizer_from_config
from vllm.transformers_utils.configs.qwen3_5 import Qwen3_5Config, Qwen3_5TextConfig
@@ -442,7 +444,7 @@ class Qwen3_5ForCausalLMBase(
def get_mrope_input_positions(
self,
input_tokens: list[int],
mm_features: list[object],
mm_features: list[MultiModalFeatureSpec],
) -> tuple[torch.Tensor, int]:
positions = torch.arange(len(input_tokens), dtype=torch.long)
return positions.unsqueeze(0).expand(3, -1), 0
@@ -488,7 +490,7 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
nn.Module.__init__(self)
config: Qwen3_5Config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config
self.model_config = vllm_config.model_config
@@ -648,6 +650,9 @@ class Qwen3_5ForConditionalGeneration(Qwen3VLForConditionalGeneration, IsHybrid)
class Qwen3_5_MoeMixtureOfExperts(MixtureOfExperts):
language_model: Qwen3_5ForCausalLM
num_local_physical_experts: int
def update_physical_experts_metadata(
self,
num_physical_experts: int,
@@ -700,14 +705,14 @@ class Qwen3_5MoeForConditionalGeneration(
Qwen3_5ForConditionalGeneration, Qwen3_5_MoeMixtureOfExperts
):
# For MoE LoRA weights loading
is_3d_moe_weight: bool = True
is_3d_moe_weight: ClassVar[bool] = True
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "model"):
# protocols have not __init__ method, so we need to use nn.Module.__init__
nn.Module.__init__(self)
config: Qwen3_5MoeConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config
self.model_config = vllm_config.model_config
+1 -1
View File
@@ -286,7 +286,7 @@ class Qwen3_5MTP(LocalArgmaxMixin, nn.Module, SupportsMultiModal, SupportsPP):
return inputs_embeds
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
+2 -1
View File
@@ -274,7 +274,7 @@ class Qwen3ASRMultiModalDataParser(MultiModalDataParser):
class Qwen3ASRMultiModalProcessor(
Qwen3OmniMoeThinkerMultiModalProcessor,
Qwen3OmniMoeThinkerMultiModalProcessor[Qwen3ASRProcessingInfo]
):
def _get_mm_fields_config(
self,
@@ -316,6 +316,7 @@ class Qwen3ASRMultiModalProcessor(
if num_features == 0:
audios = mm_items.get_items("audio", AudioProcessorItems)
audio = audios.get(item_idx)
assert audio is not None
raise ValueError(
f"The audio {audio} (len={len(audio)}) is too short "
"to be represented inside the model"
@@ -72,8 +72,9 @@ class Qwen3ASRForcedAlignerForTokenClassification(
# Remove the unused generation head created by the base class;
# the forced aligner uses a classifier head instead.
self.language_model.lm_head = None
self.language_model.logits_processor = None
language_model: nn.Module = self.language_model
language_model.lm_head = None
language_model.logits_processor = None
self.classify_num = thinker_config.classify_num
@@ -120,6 +120,7 @@ class Qwen3ASRRealtimeMultiModalProcessor(Qwen3ASRMultiModalProcessor):
)
audio_data = audios[0]
assert audio_data is not None
audio_feature_lengths = audio_data.get("audio_feature_lengths")
if audio_feature_lengths is not None:
if isinstance(audio_feature_lengths.data, torch.Tensor):
+12 -5
View File
@@ -92,7 +92,9 @@ def _dflash_layer_causal(config: Qwen3Config, layer_idx: int) -> bool:
if override is not None:
return bool(override)
layer_types = getattr(config, "layer_types", None)
return bool(layer_types) and layer_types[layer_idx] == _SLIDING_ATTENTION
if not layer_types:
return False
return layer_types[layer_idx] == _SLIDING_ATTENTION
def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
@@ -105,6 +107,7 @@ def dflash_has_any_non_causal(config: Qwen3Config) -> bool:
def _get_dflash_fc_input_size(vllm_config: VllmConfig) -> int:
spec_config = vllm_config.speculative_config
assert spec_config is not None
config = spec_config.draft_model_config.hf_config
aux_layers = get_eagle3_aux_layers_from_config(spec_config)
num_features_to_use = len(aux_layers) if aux_layers else config.num_hidden_layers
@@ -418,7 +421,9 @@ class DFlashQwen3Model(nn.Module):
prefix: str = "",
) -> None:
super().__init__()
self.config = vllm_config.speculative_config.draft_model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size
self.quant_config = get_draft_quant_config(vllm_config)
_add_global_draft_layer_exclusions(
@@ -759,7 +764,9 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.draft_model_config = vllm_config.speculative_config.draft_model_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.draft_model_config = speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
@@ -801,7 +808,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
@@ -896,7 +903,7 @@ class DFlashQwen3ForCausalLM(Qwen3ForCausalLM):
model_weights["model.mask_embedding"] = mask_embedding
self.model.has_separate_mask_embedding = True
orig_to_new_substr = {}
orig_to_new_substr: dict[str, None] = {}
if not includes_draft_id_mapping:
orig_to_new_substr["draft_id_to_target_id"] = None
if not includes_embed_tokens:
+3 -1
View File
@@ -233,7 +233,9 @@ class Qwen3DSparkModel(DFlashQwen3Model):
class Qwen3DSparkForCausalLM(DFlashQwen3ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
nn.Module.__init__(self)
self.draft_model_config = vllm_config.speculative_config.draft_model_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.draft_model_config = speculative_config.draft_model_config
self.config = self.draft_model_config.hf_config
if getattr(self.config, "draft_vocab_size", None) is None:
self.config.draft_vocab_size = getattr(self.config, "vocab_size", None)
+10 -5
View File
@@ -101,7 +101,7 @@ class Qwen3Eagle3DecoderLayer(Qwen3DecoderLayer):
hidden_states = self.hidden_norm(hidden_states)
return hidden_states, residual
def forward(
def forward( # type: ignore[override]
self,
positions: torch.Tensor,
embeds: torch.Tensor,
@@ -159,7 +159,9 @@ class Qwen3Eagle3Model(nn.Module):
prefix: str = "",
) -> None:
super().__init__()
self.config = vllm_config.speculative_config.draft_model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size
# Get drafter's quantization config
@@ -210,6 +212,7 @@ class Qwen3Eagle3Model(nn.Module):
self.config, "target_hidden_size", self.config.hidden_size
)
self.fc_input_size = target_hidden_size * num_aux_features
self.input_norm: RMSNorm | None
if self.norm_before_fc:
self.input_norm = RMSNorm(
self.fc_input_size,
@@ -282,7 +285,9 @@ class Qwen3Eagle3Model(nn.Module):
class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
# Ensure draft_vocab_size is set
# default to the base vocab size when absent
if getattr(self.config, "draft_vocab_size", None) is None:
@@ -318,7 +323,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
else:
self.draft_id_to_target_id = None
self.use_parallel_drafting = vllm_config.speculative_config.parallel_drafting
self.use_parallel_drafting = speculative_config.parallel_drafting
if self.use_parallel_drafting:
self.register_buffer(
@@ -335,7 +340,7 @@ class Eagle3Qwen3ForCausalLM(Qwen3ForCausalLM):
) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
+9 -6
View File
@@ -177,6 +177,8 @@ class Qwen3MoeSparseMoeBlock(nn.Module):
shared_expert_intermediate_size = getattr(
config, "shared_expert_intermediate_size", 0
)
self.shared_expert_gate: ReplicatedLinear | None
self.shared_expert: Qwen3MoeMLP | None
if shared_expert_intermediate_size > 0:
self.shared_expert_gate = ReplicatedLinear(
config.hidden_size,
@@ -302,6 +304,12 @@ class Qwen3MoeAttention(nn.Module):
rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config,
)
attention_kwargs: dict[str, Any] = {}
if dual_chunk_attention_config:
attention_kwargs = {
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
self.attn = Attention(
self.num_heads,
self.head_dim,
@@ -310,12 +318,7 @@ class Qwen3MoeAttention(nn.Module):
cache_config=cache_config,
quant_config=quant_config,
prefix=f"{prefix}.attn",
**{
"layer_idx": extract_layer_index(prefix),
"dual_chunk_attention_config": dual_chunk_attention_config,
}
if dual_chunk_attention_config
else {},
**attention_kwargs,
)
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
+3
View File
@@ -757,6 +757,9 @@ class Qwen3NextModel(nn.Module, EagleModelMixin):
class QwenNextMixtureOfExperts(MixtureOfExperts):
model: Qwen3NextModel
num_local_physical_experts: int
def update_physical_experts_metadata(
self,
num_physical_experts: int,
@@ -24,7 +24,7 @@
from collections.abc import Callable, Iterable, Iterator, Mapping, Sequence
from functools import partial
from typing import Any, cast
from typing import Any, ClassVar
import numpy as np
import torch
@@ -41,12 +41,13 @@ from transformers.models.qwen3_omni_moe.processing_qwen3_omni_moe import (
Qwen3OmniMoeProcessor,
)
from transformers.models.whisper import WhisperFeatureExtractor
from typing_extensions import TypeVar
from vllm.compilation.decorators import support_torch_compile
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed import get_pp_group, get_tensor_model_parallel_world_size
from vllm.inputs import PromptType
from vllm.inputs import PromptType, TextPrompt
from vllm.logger import init_logger
from vllm.model_executor.layers.activation import _ACTIVATION_REGISTRY
from vllm.model_executor.layers.attention.mm_encoder_attention import (
@@ -73,6 +74,7 @@ from vllm.multimodal.inputs import (
)
from vllm.multimodal.parse import AudioProcessorItems, MultiModalDataItems
from vllm.multimodal.processing.processor import (
BaseProcessingInfo,
HFMultiModalInputs,
MultiModalProcessingResult,
MultiModalPromptUpdates,
@@ -102,12 +104,15 @@ from .qwen2_5_omni_thinker import (
Qwen2_5OmniThinkerDummyInputsBuilder,
Qwen2_5OmniThinkerMultiModalDataParser,
Qwen2_5OmniThinkerMultiModalProcessor,
_get_feature_tensor,
check_interleaved_audio_video,
merge_interleaved_embeddings,
)
from .qwen2_5_vl import (
Qwen2_5_VisionAttention,
Qwen2_5_VLImageInputs,
Qwen2_5_VLProcessingInfo,
Qwen2_5_VLVideoInputs,
)
from .qwen3_moe import Qwen3MoeForCausalLM, Qwen3MoeModel
from .utils import (
@@ -963,7 +968,7 @@ class Qwen3Omni_VisionTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
grid_thw: list[list[int]],
grid_thw: torch.Tensor,
) -> torch.Tensor:
hidden_states = x.to(device=self.device, dtype=self.dtype)
hidden_states = self.patch_embed(hidden_states)
@@ -1201,7 +1206,7 @@ class Qwen3OmniMoeThinkerProcessingInfo(
self,
seq_len: int,
mm_counts: Mapping[str, int] | None = None,
) -> Mapping[str, int] | None:
) -> Mapping[str, int]:
mm_counts = mm_counts or {}
requested_modalities = {m for m, c in mm_counts.items() if c > 0}
mm_max_tokens: dict[str, int] = {}
@@ -1231,11 +1236,15 @@ class Qwen3OmniMoeThinkerProcessingInfo(
return mm_max_tokens
Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder
Qwen3OmniMoeThinkerDummyInputsBuilder = Qwen2_5OmniThinkerDummyInputsBuilder[
Qwen3OmniMoeThinkerProcessingInfo
]
_I = TypeVar("_I", bound=BaseProcessingInfo, default=Qwen3OmniMoeThinkerProcessingInfo)
class Qwen3OmniMoeThinkerMultiModalProcessor(
Qwen2_5OmniThinkerMultiModalProcessor,
Qwen2_5OmniThinkerMultiModalProcessor[_I],
):
def _get_hf_mm_inputs(
self,
@@ -1246,7 +1255,8 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
audios = hf_inputs.hf_data.get("audio")
if audios:
feature_extractor = self.info.get_feature_extractor()
assert isinstance(audios, (list, tuple))
feature_extractor = self.info.get_hf_processor().feature_extractor
hop_length = feature_extractor.hop_length
def pad_to_hop_length(x: np.ndarray) -> np.ndarray:
@@ -1283,9 +1293,10 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
):
audios = mm_data.get("audio", [])
if audios:
feature_extractor = self.info.get_feature_extractor(
assert isinstance(audios, (list, tuple))
feature_extractor = self.info.get_hf_processor(
**hf_processor_mm_kwargs
)
).feature_extractor
hop_length = feature_extractor.hop_length
audio_num_frames = []
@@ -1421,6 +1432,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
) -> Sequence[PromptUpdate]:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
tokenizer = self.info.get_tokenizer()
assert isinstance(self.info, Qwen2_5_VLProcessingInfo)
image_processor = self.info.get_image_processor(**hf_processor_mm_kwargs)
vocab = tokenizer.get_vocab()
@@ -1460,6 +1472,7 @@ class Qwen3OmniMoeThinkerMultiModalProcessor(
if num_features == 0:
audios = mm_items.get_items("audio", AudioProcessorItems)
audio = audios.get(item_idx)
assert audio is not None
raise ValueError(
f"The audio {audio} (len={len(audio)}) is too short "
"to be represented inside the model"
@@ -1607,8 +1620,8 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
Qwen3OmniMoeConditionalGenerationMixin,
SupportsTranscription,
):
is_3d_moe_weight: bool = True
supports_tower_connector_lora: bool = True
is_3d_moe_weight: ClassVar[bool] = True
supports_tower_connector_lora: ClassVar[bool] = True
hf_to_vllm_mapper = WeightsMapper(
orig_to_new_prefix={
"thinker.lm_head.": "language_model.lm_head.",
@@ -1769,7 +1782,13 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
self.deepstack_input_embeds_num_tokens = 0
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
mm_input_by_modality: dict[
str,
Qwen2_5_VLImageInputs
| Qwen2_5_VLVideoInputs
| Qwen2_5OmniAudioFeatureInputs
| None,
] = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -1862,6 +1881,7 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
if self.visual.deepstack_visual_indexes is not None and any(
has_vision_embeddings
):
multimodal_embeddings = list(multimodal_embeddings)
multiscale_len = len(self.visual.deepstack_visual_indexes)
multimodal_embeddings_multiscale = []
@@ -2006,8 +2026,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
f
for f in mm_features
if f.modality == "video"
and f.data is not None
and f.data.get("use_audio_in_video")
and f.data["use_audio_in_video"].data.item()
and _get_feature_tensor(f, "use_audio_in_video").item()
]
audios = [f for f in mm_features if f.modality == "audio"]
@@ -2015,7 +2036,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
paired_audio_offsets: set[int] = set()
for i, video_f in enumerate(videos_with_audio):
if i < len(audios):
audio_len = audios[i].data["audio_feature_lengths"].data.item()
audio_len = int(
_get_feature_tensor(audios[i], "audio_feature_lengths").item()
)
mapping[video_f.mm_position.offset] = audio_len
paired_audio_offsets.add(audios[i].mm_position.offset)
return mapping, paired_audio_offsets
@@ -2043,9 +2066,10 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
for mm_feature in sorted_features:
offset = mm_feature.mm_position.offset
modality = mm_feature.modality
assert mm_feature.data is not None
if modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
t, h, w = _get_feature_tensor(mm_feature, "image_grid_thw").tolist()
yield (
offset,
"image",
@@ -2057,15 +2081,15 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
},
)
elif modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
t, h, w = _get_feature_tensor(mm_feature, "video_grid_thw").tolist()
second_per_grid_ts = 2.0
if mm_feature.data.get("second_per_grid_ts"):
second_per_grid_ts = mm_feature.data[
"second_per_grid_ts"
].data.item()
second_per_grid_ts = _get_feature_tensor(
mm_feature, "second_per_grid_ts"
).item()
use_audio_in_video = bool(
mm_feature.data.get("use_audio_in_video")
and mm_feature.data["use_audio_in_video"].data.item()
and _get_feature_tensor(mm_feature, "use_audio_in_video").item()
)
yield (
@@ -2082,7 +2106,9 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
)
elif modality == "audio":
if offset not in paired_audio_offsets:
audio_len = mm_feature.data["audio_feature_lengths"].data.item()
audio_len = _get_feature_tensor(
mm_feature, "audio_feature_lengths"
).item()
yield offset, "audio", {"audio_feature_length": audio_len}
def _compute_interleaved_positions(
@@ -2170,8 +2196,14 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
to_language = "en"
# Get full language names from supported_languages mapping
full_lang_name = cls.supported_languages.get(language, "")
full_lang_name_to = cls.supported_languages.get(to_language, "")
full_lang_name = (
cls.supported_languages.get(language, "") if language is not None else ""
)
full_lang_name_to = (
cls.supported_languages.get(to_language, "")
if to_language is not None
else ""
)
if task_type == "transcribe" and full_lang_name:
instruction += f" into {full_lang_name}"
@@ -2201,8 +2233,11 @@ class Qwen3OmniMoeThinkerForConditionalGeneration(
)
audio_data = (audio, stt_config.sample_rate)
prompts_dict = {"multi_modal_data": {"audio": audio_data}, "prompt": prompt}
return cast(PromptType, prompts_dict)
prompts_dict: TextPrompt = {
"multi_modal_data": {"audio": audio_data},
"prompt": prompt,
}
return prompts_dict
def get_mrope_input_positions(
self,
+34 -16
View File
@@ -582,7 +582,9 @@ class Qwen3_VisionTransformer(nn.Module):
else []
)
self.num_grid_per_side = int(self.num_position_embeddings**0.5)
self._rot_pos_ids_cache = LRUCache(capacity=1024)
self._rot_pos_ids_cache: LRUCache[tuple[int, int, int], torch.Tensor] = (
LRUCache(capacity=1024)
)
use_data_parallel = is_vit_use_data_parallel()
self.tp_size = (
@@ -705,7 +707,7 @@ class Qwen3_VisionTransformer(nn.Module):
self._rot_pos_ids_cache[cache_key] = result
return result
def rot_pos_emb(self, grid_thw: list[list[int]]):
def rot_pos_emb(self, grid_thw: Sequence[Sequence[int]]):
max_grid_size = max(max(h, w) for _, h, w in grid_thw)
pos_ids = [
self.rot_pos_ids(h, w, self.spatial_merge_size)
@@ -731,7 +733,9 @@ class Qwen3_VisionTransformer(nn.Module):
return cos_combined, sin_combined
def fast_pos_embed_interpolate(self, grid_thw: list[list[int]]) -> torch.Tensor:
def fast_pos_embed_interpolate(
self, grid_thw: Sequence[Sequence[int]]
) -> torch.Tensor:
interpolate_fn = (
triton_pos_embed_interpolate if HAS_TRITON else pos_embed_interpolate_native
)
@@ -752,7 +756,7 @@ class Qwen3_VisionTransformer(nn.Module):
def prepare_encoder_metadata(
self,
grid_thw_list: list[list[int]],
grid_thw_list: Sequence[Sequence[int]],
*,
max_batch_size: int | None = None,
max_frames_per_batch: int | None = None,
@@ -1005,9 +1009,10 @@ class Qwen3VLProcessingInfo(Qwen2VLProcessingInfo):
self,
seq_len: int,
mm_counts: Mapping[str, int],
max_frames_per_video: int = DUMMY_VIDEO_NUM_FRAMES,
) -> int:
return super().get_num_frames_with_most_features(
seq_len, mm_counts, max_frames_per_video=DUMMY_VIDEO_NUM_FRAMES
seq_len, mm_counts, max_frames_per_video=max_frames_per_video
)
def get_max_video_tokens(
@@ -1253,7 +1258,7 @@ class Qwen3VLDummyInputsBuilder(BaseDummyInputsBuilder[Qwen3VLProcessingInfo]):
)
videos = [v.copy() for v in videos]
video_items = []
video_items: list[VideoItem] = []
for video in videos:
video_num_frames = video.shape[0]
video_metadata = {
@@ -1339,6 +1344,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# are processed into several image patches
video_input_ids_lst: list[list[int]] = []
if videos := hf_data.pop("videos", []):
assert isinstance(videos, list)
video_grid_thw_lst = []
pixel_values_videos_lst = []
timestamps_per_video = []
@@ -1361,7 +1367,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
# NOTE: a copy of is created to update do_sample_frames,
# otherwise mm_hash for the object will be incorrect.
video_mm_kwargs = dict(**hf_kwargs)
video_mm_kwargs: dict[str, Any] = dict(hf_kwargs)
merged = self.info.ctx.get_merged_mm_kwargs(hf_kwargs, modality="video")
if merged.keys() & {"size", "min_pixels", "max_pixels"}:
video_size = dict(self.info.get_video_processor().size)
@@ -1551,6 +1557,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
sampled_fps = sampled_fps[item_idx]
timestamps = out_item["timestamps"].data
assert isinstance(timestamps, (list, torch.Tensor))
assert len(timestamps) == grid_thw[0], (
f"The timestamps length({len(timestamps)}) should be equal "
f"video length ({grid_thw[0]})."
@@ -1617,7 +1624,7 @@ class Qwen3VLMultiModalProcessor(BaseMultiModalProcessor[Qwen3VLProcessingInfo])
def get_video_repl(
*,
tokens_per_frame: list[int],
timestamps: list[float | int] | torch.Tensor,
timestamps: Sequence[float] | torch.Tensor,
tokenizer: TokenizerLike,
vision_start_token_id: int,
vision_end_token_id: int,
@@ -1842,7 +1849,7 @@ class Qwen3VLForConditionalGeneration(
super().__init__()
config: Qwen3VLConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config
self.model_config = vllm_config.model_config
@@ -2006,6 +2013,7 @@ class Qwen3VLForConditionalGeneration(
def get_max_frames_per_video(self) -> int:
mm_registry = MULTIMODAL_REGISTRY
info = mm_registry.get_processing_info(self.model_config)
assert isinstance(info, Qwen3VLProcessingInfo)
max_frames_per_video = info.get_num_frames_with_most_features(
seq_len=self.model_config.max_model_len,
mm_counts={"video": self.multimodal_config.get_limit_per_prompt("video")},
@@ -2254,7 +2262,7 @@ class Qwen3VLForConditionalGeneration(
image_grid_thw=image_grid_thw,
)
if image_embeds is not None:
else:
return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds",
image_embeds=image_embeds,
@@ -2282,7 +2290,7 @@ class Qwen3VLForConditionalGeneration(
timestamps=timestamps,
)
if video_embeds is not None:
else:
return Qwen2_5_VLVideoEmbeddingInputs(
type="video_embeds",
video_embeds=video_embeds,
@@ -2405,6 +2413,7 @@ class Qwen3VLForConditionalGeneration(
# Apply EVS to each video.
video_embeds_out = []
assert video_input.timestamps is not None
for video_idx, (emb, size) in enumerate(zip(video_embeds_split, grid_thw_list)):
# Compute positions.
timestamps = video_input.timestamps[video_idx]
@@ -2412,6 +2421,7 @@ class Qwen3VLForConditionalGeneration(
t, h, w = size
if self.is_multimodal_pruning_enabled:
assert self.video_pruning_rate is not None
# Compute the retention mask for each video (EVS or VidCom2).
if self.video_pruning_method == "vidcom2":
mask_fn = vidcom2_compute_retention_mask
@@ -2583,7 +2593,7 @@ class Qwen3VLForConditionalGeneration(
{
"video_grid_thw": MultiModalFieldElem(
data=torch.tensor(video_grid_thw),
field=None, # HACK.
field=MultiModalFieldConfig.batched("video").field,
),
}
),
@@ -2611,7 +2621,9 @@ class Qwen3VLForConditionalGeneration(
return expanded_positions
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
mm_input_by_modality: dict[
str, Qwen2_5_VLImageInputs | Qwen2_5_VLVideoInputs | None
] = {}
for input_key in kwargs:
if (
input_key in ("pixel_values", "image_embeds")
@@ -2659,14 +2671,20 @@ class Qwen3VLForConditionalGeneration(
"""
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset
data = mm_feature.data
assert data is not None
if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
grid = data["image_grid_thw"].data
assert isinstance(grid, torch.Tensor)
t, h, w = grid.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
llm_grid_h = h // spatial_merge_size
llm_grid_w = w // spatial_merge_size
yield offset, llm_grid_h, llm_grid_w, llm_grid_h * llm_grid_w
elif mm_feature.modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
grid = data["video_grid_thw"].data
assert isinstance(grid, torch.Tensor)
t, h, w = grid.tolist()
llm_grid_h = h // spatial_merge_size
llm_grid_w = w // spatial_merge_size
@@ -2721,7 +2739,7 @@ class Qwen3VLForConditionalGeneration(
mm_features: list[MultiModalFeatureSpec],
config: Qwen3VLConfig,
):
llm_pos_ids_list = []
llm_pos_ids_list: list[np.ndarray] = []
st = 0
for (
offset,
+8 -4
View File
@@ -25,6 +25,7 @@
"""Inference-only Qwen3-VL-MoE model compatible with HuggingFace weights."""
from itertools import islice
from typing import ClassVar
import torch
from transformers.models.qwen3_vl_moe.configuration_qwen3_vl_moe import (
@@ -176,6 +177,9 @@ class Qwen3MoeLLMForCausalLM(Qwen3MoeForCausalLM):
class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
language_model: Qwen3MoeLLMForCausalLM
num_local_physical_experts: int
def update_physical_experts_metadata(
self,
num_physical_experts: int,
@@ -223,7 +227,7 @@ class Qwen3VLMoeMixtureOfExperts(MixtureOfExperts):
class Qwen3VLMoeForConditionalGeneration(
Qwen3VLForConditionalGeneration, Qwen3VLMoeMixtureOfExperts
):
is_3d_moe_weight: bool = True
is_3d_moe_weight: ClassVar[bool] = True
packed_modules_mapping = {
"qkv_proj": [
"q_proj",
@@ -236,7 +240,7 @@ class Qwen3VLMoeForConditionalGeneration(
super(Qwen3VLForConditionalGeneration, self).__init__()
config: Qwen3VLMoeConfig = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config
multimodal_config = vllm_config.model_config.get_multimodal_config()
self.config = config
self.model_config = vllm_config.model_config
@@ -293,8 +297,8 @@ class Qwen3VLMoeForConditionalGeneration(
)
# Whether to include the gate_up_proj mapping is determined by
# the language model.
self.packed_modules_mapping = (
# the language model. Keep this override local to the instance.
self.packed_modules_mapping = ( # type: ignore[misc]
self.packed_modules_mapping | self.language_model.packed_modules_mapping
)