[Mypy] Fix typing for M models (#54262)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
This commit is contained in:
Taneem Ibrahim
2026-09-01 11:26:00 -04:00
committed by GitHub
parent d9eb4e344f
commit 76f3249fbd
26 changed files with 512 additions and 200 deletions
-1
View File
@@ -114,7 +114,6 @@ SEPARATE_GROUPS = [
EXCLUDE = [ EXCLUDE = [
r"vllm/model_executor/models/[kK]", r"vllm/model_executor/models/[kK]",
r"vllm/model_executor/models/[lL]", r"vllm/model_executor/models/[lL]",
r"vllm/model_executor/models/[mM]",
r"vllm/model_executor/models/[nN]", r"vllm/model_executor/models/[nN]",
r"vllm/model_executor/models/[oO]", r"vllm/model_executor/models/[oO]",
r"vllm/model_executor/models/[pP]", r"vllm/model_executor/models/[pP]",
+2 -2
View File
@@ -52,14 +52,14 @@ class MambaDecoderLayer(nn.Module):
model_config: ModelConfig | None = None, model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None, cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None, quant_config: QuantizationConfig | None = None,
is_lora_enabled: bool | None = False, is_lora_enabled: bool = False,
prefix: str = "", prefix: str = "",
) -> None: ) -> None:
super().__init__() super().__init__()
self.config = config self.config = config
self.is_falcon_mamba = config.model_type == "falcon_mamba" self.is_falcon_mamba = config.model_type == "falcon_mamba"
self.is_lora_enabled = is_lora_enabled self.is_lora_enabled = is_lora_enabled
mixer_rms_eps = config.mixer_rms_eps if self.is_falcon_mamba else None mixer_rms_eps = config.mixer_rms_eps if self.is_falcon_mamba else 1e-5
self.mixer = MambaMixer( self.mixer = MambaMixer(
hidden_size=config.hidden_size, hidden_size=config.hidden_size,
ssm_state_size=config.state_size, ssm_state_size=config.state_size,
+3 -1
View File
@@ -54,7 +54,9 @@ class Medusa(nn.Module):
needs to have truncated_vocab_size (=k) as an attribute.""" needs to have truncated_vocab_size (=k) as an attribute."""
def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None: def __init__(self, *, vllm_config: VllmConfig, prefix: str = "") -> None:
config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
super().__init__() super().__init__()
self.config = config self.config = config
self.blocks = nn.ModuleList( self.blocks = nn.ModuleList(
+24 -18
View File
@@ -92,24 +92,30 @@ class MellumAttention(Qwen3MoeAttention):
rope_parameters=rope_parameters, rope_parameters=rope_parameters,
dual_chunk_attention_config=dual_chunk_attention_config, dual_chunk_attention_config=dual_chunk_attention_config,
) )
self.attn = Attention( if dual_chunk_attention_config:
self.num_heads, self.attn = Attention(
self.head_dim, self.num_heads,
self.scaling, self.head_dim,
num_kv_heads=self.num_kv_heads, self.scaling,
cache_config=cache_config, num_kv_heads=self.num_kv_heads,
quant_config=quant_config, cache_config=cache_config,
per_layer_sliding_window=per_layer_sliding_window, quant_config=quant_config,
prefix=f"{prefix}.attn", per_layer_sliding_window=per_layer_sliding_window,
**( prefix=f"{prefix}.attn",
{ layer_idx=extract_layer_index(prefix),
"layer_idx": extract_layer_index(prefix), dual_chunk_attention_config=dual_chunk_attention_config,
"dual_chunk_attention_config": dual_chunk_attention_config, )
} else:
if dual_chunk_attention_config self.attn = Attention(
else {} self.num_heads,
), self.head_dim,
) self.scaling,
num_kv_heads=self.num_kv_heads,
cache_config=cache_config,
quant_config=quant_config,
per_layer_sliding_window=per_layer_sliding_window,
prefix=f"{prefix}.attn",
)
self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.q_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps) self.k_norm = RMSNorm(self.head_dim, eps=rms_norm_eps)
+11 -5
View File
@@ -36,7 +36,7 @@ from torch.nn.functional import scaled_dot_product_attention
from transformers import BatchFeature from transformers import BatchFeature
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.distributed import get_tensor_model_parallel_world_size from vllm.distributed import get_tensor_model_parallel_world_size
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
from vllm.model_executor.layers.activation import get_act_fn from vllm.model_executor.layers.activation import get_act_fn
@@ -570,6 +570,7 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return { return {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
@@ -593,6 +594,7 @@ class MiDashengLMMultiModalProcessor(
) -> tuple[Mapping[str, object], Mapping[str, object]]: ) -> tuple[Mapping[str, object], Mapping[str, object]]:
mm_data = dict(mm_data) mm_data = dict(mm_data)
audios = mm_data.pop("audios", []) audios = mm_data.pop("audios", [])
assert isinstance(audios, list)
min_audio_len = self.info.get_min_audio_len() min_audio_len = self.info.get_min_audio_len()
processed_audios = [ processed_audios = [
@@ -637,6 +639,7 @@ class MiDashengLMMultiModalProcessor(
out_mm_data = out_mm_kwargs.get_data() out_mm_data = out_mm_kwargs.get_data()
audio_length = out_mm_data.get("audio_length") audio_length = out_mm_data.get("audio_length")
audio_output_lengths: list[int]
if audio_length is None: if audio_length is None:
audio_output_lengths = [] audio_output_lengths = []
else: else:
@@ -645,10 +648,13 @@ class MiDashengLMMultiModalProcessor(
if isinstance(audio_length, torch.Tensor) if isinstance(audio_length, torch.Tensor)
else audio_length else audio_length
) )
audio_output_lengths = [ assert isinstance(audio_length_np, (list, tuple, np.ndarray))
max(1, calculate_mel_frames_dasheng(int(length))) # at least one frame audio_output_lengths = []
for length in audio_length_np for length in audio_length_np:
] assert isinstance(length, (int, np.integer))
audio_output_lengths.append(
max(1, calculate_mel_frames_dasheng(int(length)))
)
def get_replacement_midashenglm(item_idx: int): def get_replacement_midashenglm(item_idx: int):
num_features = audio_output_lengths[item_idx] num_features = audio_output_lengths[item_idx]
+18 -11
View File
@@ -531,18 +531,18 @@ class MiMoAudioTokenizerConfig(PretrainedConfig):
kernel_size: int = 3, kernel_size: int = 3,
activation_function: str = "gelu", activation_function: str = "gelu",
encoder_layers: int = 8, encoder_layers: int = 8,
encoder_skip_layer_id: int = None, encoder_skip_layer_id: int | None = None,
encoder_attention_heads: int = 12, encoder_attention_heads: int = 12,
encoder_ffn_dim: int = 3072, encoder_ffn_dim: int = 3072,
encoder_causal: bool = False, encoder_causal: bool = False,
encoder_attn_window_size: list = None, encoder_attn_window_size: list[int] | None = None,
decoder_layers: int = 8, decoder_layers: int = 8,
decoder_attention_heads: int = 12, decoder_attention_heads: int = 12,
decoder_ffn_dim: int = 3072, decoder_ffn_dim: int = 3072,
decoder_kernel_size: int = 3, decoder_kernel_size: int = 3,
decoder_stride_size: int = 2, decoder_stride_size: int = 2,
decoder_causal: bool = True, decoder_causal: bool = True,
decoder_attn_window_size: list = None, decoder_attn_window_size: list[int] | None = None,
nfft: int = 1024, nfft: int = 1024,
vocoder_dim: int = 512, vocoder_dim: int = 512,
vocoder_intermediate_dim: int = 4096, vocoder_intermediate_dim: int = 4096,
@@ -553,16 +553,16 @@ class MiMoAudioTokenizerConfig(PretrainedConfig):
window_size: int = 1024, window_size: int = 1024,
vocoder_padding: str = "same", vocoder_padding: str = "same",
fmin: int = 0, fmin: int = 0,
fmax: int = None, fmax: int | None = None,
num_quantizers: int = 12, num_quantizers: int = 12,
codebook_size: list = None, codebook_size: list[int] | None = None,
threshold_ema_dead_code: int = 10, threshold_ema_dead_code: int = 10,
position_embedding_type: str = "rope", position_embedding_type: str = "rope",
rope_theta: int = 10000, rope_theta: int = 10000,
rope_type: str = "default", rope_type: str = "default",
ln_type: str = "LayerNorm", ln_type: str = "LayerNorm",
vocoder_attention_heads: int = 4, vocoder_attention_heads: int = 4,
vocoder_attn_window_size: list = None, vocoder_attn_window_size: list[int] | None = None,
use_istft_only: bool = False, use_istft_only: bool = False,
hybrid_attention: bool = False, hybrid_attention: bool = False,
hybrid_block_size: int = 8, hybrid_block_size: int = 8,
@@ -808,17 +808,22 @@ class AudioEncoder(nn.Module):
config.rope_type, config.rope_type,
) )
attn_window_sizes = [] encoder_attn_window_size = config.encoder_attn_window_size
assert encoder_attn_window_size is not None
assert len(encoder_attn_window_size) == 2
attn_window_size = (
encoder_attn_window_size[0],
encoder_attn_window_size[1],
)
attn_window_sizes: list[tuple[int, int]] = []
if config.hybrid_attention: if config.hybrid_attention:
for i in range(config.encoder_layers): for i in range(config.encoder_layers):
if i % config.swa_per_block < config.swa_per_block - 1: if i % config.swa_per_block < config.swa_per_block - 1:
attn_window_sizes.append(tuple(config.encoder_attn_window_size)) attn_window_sizes.append(attn_window_size)
else: else:
attn_window_sizes.append((-1, -1)) attn_window_sizes.append((-1, -1))
else: else:
attn_window_sizes = [ attn_window_sizes = [attn_window_size] * config.encoder_layers
tuple(config.encoder_attn_window_size)
] * config.encoder_layers
self.layers = nn.ModuleList( self.layers = nn.ModuleList(
[ [
@@ -848,6 +853,7 @@ class AudioEncoder(nn.Module):
else: else:
self.down_sample_layer = None self.down_sample_layer = None
self.quantizer: ResidualVectorQuantizer | None
if config.num_quantizers != 0: if config.num_quantizers != 0:
self.quantizer = ResidualVectorQuantizer( self.quantizer = ResidualVectorQuantizer(
dimension=config.d_model, dimension=config.d_model,
@@ -976,6 +982,7 @@ class AudioEncoder(nn.Module):
@torch.no_grad() @torch.no_grad()
def decode_vq(self, codes): def decode_vq(self, codes):
assert self.quantizer is not None
self.quantizer.float() self.quantizer.float()
return self.quantizer.decode(codes) return self.quantizer.decode(codes)
+5 -3
View File
@@ -303,6 +303,7 @@ class MiMoV2Attention(nn.Module):
backend_enum = requested backend_enum = requested
else: else:
fa_backend = AttentionBackendEnum.FLASH_ATTN_DIFFKV.get_class() fa_backend = AttentionBackendEnum.FLASH_ATTN_DIFFKV.get_class()
assert hasattr(fa_backend, "is_supported_on_current_device")
if fa_backend.is_supported_on_current_device( if fa_backend.is_supported_on_current_device(
head_size=self.head_dim, head_size=self.head_dim,
head_size_v=self.v_head_dim, head_size_v=self.v_head_dim,
@@ -312,6 +313,7 @@ class MiMoV2Attention(nn.Module):
else: else:
backend_enum = AttentionBackendEnum.TRITON_ATTN_DIFFKV backend_enum = AttentionBackendEnum.TRITON_ATTN_DIFFKV
attn_backend = backend_enum.get_class() attn_backend = backend_enum.get_class()
assert hasattr(attn_backend, "set_head_size_v")
attn_backend.set_head_size_v(self.v_head_dim) attn_backend.set_head_size_v(self.v_head_dim)
logger.info_once("Using %s for attention.", attn_backend.get_name()) logger.info_once("Using %s for attention.", attn_backend.get_name())
else: else:
@@ -636,7 +638,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
) )
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]: def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
stacked_params_mapping = [ stacked_params_mapping: list[tuple[str, str, str | int]] = [
# (param_name, shard_name, shard_id) # (param_name, shard_name, shard_id)
("qkv_proj", "q_proj", "q"), ("qkv_proj", "q_proj", "q"),
("qkv_proj", "k_proj", "k"), ("qkv_proj", "k_proj", "k"),
@@ -709,7 +711,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
): ):
continue continue
stacked_matched = False stacked_matched = False
for param_name, weight_name, shard_id in stacked_params_mapping: for param_name, weight_name, stacked_shard_id in stacked_params_mapping:
if weight_name not in name: if weight_name not in name:
continue continue
name_rewritten = name.replace(weight_name, param_name) name_rewritten = name.replace(weight_name, param_name)
@@ -728,7 +730,7 @@ class MiMoV2Model(nn.Module, EagleModelMixin):
param = params_dict[name_rewritten] param = params_dict[name_rewritten]
weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader(param, loaded_weight, shard_id) weight_loader(param, loaded_weight, stacked_shard_id)
loaded_params.add(name_rewritten) loaded_params.add(name_rewritten)
stacked_matched = True stacked_matched = True
+10 -5
View File
@@ -255,7 +255,7 @@ class MiMoV2MTP(nn.Module):
tp_rank = get_tensor_model_parallel_rank() tp_rank = get_tensor_model_parallel_rank()
tp_size = get_tensor_model_parallel_world_size() tp_size = get_tensor_model_parallel_world_size()
stacked_params_mapping = [ stacked_params_mapping: list[tuple[str, str, str | int]] = [
("gate_up_proj", "gate_proj", 0), ("gate_up_proj", "gate_proj", 0),
("gate_up_proj", "up_proj", 1), ("gate_up_proj", "up_proj", 1),
# Flash format: separate projections → fused qkv_proj # Flash format: separate projections → fused qkv_proj
@@ -308,8 +308,11 @@ class MiMoV2MTP(nn.Module):
if name_rewritten not in params_dict: if name_rewritten not in params_dict:
continue continue
param = params_dict[name_rewritten] param = params_dict[name_rewritten]
weight_loader = getattr(param, "weight_loader", default_weight_loader) weight_loader = getattr(param, "weight_loader", None)
weight_loader(param, loaded_weight, shard_id) if weight_loader is None:
default_weight_loader(param, loaded_weight)
else:
weight_loader(param, loaded_weight, shard_id)
loaded_params.add(name_rewritten) loaded_params.add(name_rewritten)
stacked_matched = True stacked_matched = True
break break
@@ -331,8 +334,10 @@ class MiMoV2MTP(nn.Module):
0, tp_rank * heads_per_rank, heads_per_rank 0, tp_rank * heads_per_rank, heads_per_rank
) )
weight_loader = getattr(param, "weight_loader", default_weight_loader) direct_weight_loader = getattr(
weight_loader(param, loaded_weight) param, "weight_loader", default_weight_loader
)
direct_weight_loader(param, loaded_weight)
loaded_params.add(name) loaded_params.add(name)
return loaded_params return loaded_params
+119 -43
View File
@@ -12,9 +12,14 @@ import torch.nn as nn
import torch.nn.functional as F import torch.nn.functional as F
from transformers import BatchFeature, PretrainedConfig from transformers import BatchFeature, PretrainedConfig
from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize from transformers.models.qwen2_vl.image_processing_qwen2_vl import smart_resize
from typing_extensions import TypedDict
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import parallel_state from vllm.distributed import parallel_state
from vllm.distributed import utils as dist_utils from vllm.distributed import utils as dist_utils
from vllm.inputs import MultiModalDataDict from vllm.inputs import MultiModalDataDict
@@ -70,6 +75,18 @@ from .qwen2_vl import _create_qwen2vl_field_factory
from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix from .utils import AutoWeightsLoader, IntermediateTensors, WeightsMapper, maybe_prefix
class MiMoV2OmniAudioInputs(TypedDict):
type: str
audio_features: torch.Tensor | list[torch.Tensor]
audio_token_lens: torch.Tensor | list[int] | None
class MiMoV2OmniMultiModalInputs(TypedDict, total=False):
image: Qwen2_5_VLImageInputs | None
video: Qwen2_5_VLVideoInputs | None
audio: MiMoV2OmniAudioInputs | None
class MiMoVisionMLP(Qwen2_5_VisionMLP): class MiMoVisionMLP(Qwen2_5_VisionMLP):
pass pass
@@ -937,11 +954,14 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
# Handle video_audio items: convert video part to (TCHW, timestamps) tuple # Handle video_audio items: convert video part to (TCHW, timestamps) tuple
if "video_audio" in mm_data: if "video_audio" in mm_data:
va_converted: list[VideoAudioInput] = [] va_converted: list[VideoAudioInput] = []
for va_item in mm_data["video_audio"]: video_audio_items = mm_data["video_audio"]
assert isinstance(video_audio_items, Sequence)
for va_item in video_audio_items:
if isinstance(va_item, VideoAudioInput): if isinstance(va_item, VideoAudioInput):
vid = va_item.video vid = va_item.video
else: else:
# Expect (video_frames, audio_source) tuple # Expect (video_frames, audio_source) tuple
assert isinstance(va_item, (list, tuple)) and len(va_item) == 2
vid, audio_src = va_item vid, audio_src = va_item
va_item = VideoAudioInput(video=vid, audio=audio_src) va_item = VideoAudioInput(video=vid, audio=audio_src)
vid = vid vid = vid
@@ -976,7 +996,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
if "videos" in mm_data: if "videos" in mm_data:
converted: list[tuple[torch.Tensor, torch.Tensor]] = [] converted: list[tuple[torch.Tensor, torch.Tensor]] = []
for video in mm_data["videos"]: videos = mm_data["videos"]
assert isinstance(videos, Sequence)
for video in videos:
if ( if (
isinstance(video, tuple) isinstance(video, tuple)
and len(video) == 2 and len(video) == 2
@@ -1037,30 +1059,47 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
video_end_id = p.video_end_token_id video_end_id = p.video_end_token_id
audio_start_id = p.audio_start_token_id audio_start_id = p.audio_start_token_id
audio_end_id = p.audio_end_token_id audio_end_id = p.audio_end_token_id
assert isinstance(vision_start_id, int)
assert isinstance(vision_end_id, int)
assert isinstance(video_start_id, int)
assert isinstance(video_end_id, int)
def get_image_replacement(item_idx: int) -> PromptUpdateDetails: def get_image_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["image"][item_idx] out_item = out_mm_kwargs["image"][item_idx]
grid_thw = out_item["image_grid_thw"].data grid_thw = out_item["image_grid_thw"].data
assert isinstance(grid_thw, torch.Tensor)
n_tokens = int(grid_thw.prod()) // merge_size**2 n_tokens = int(grid_thw.prod()) // merge_size**2
return [image_pad_id] * n_tokens return PromptUpdateDetails.from_seq([image_pad_id] * n_tokens)
def get_video_replacement(item_idx: int) -> PromptUpdateDetails: def get_video_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["video"][item_idx] out_item = out_mm_kwargs["video"][item_idx]
grid_thw = out_item["video_grid_thw"].data grid_thw = out_item["video_grid_thw"].data
spt = float(out_item["second_per_grid_ts"].data) second_per_grid_ts = out_item["second_per_grid_ts"].data
start = float(out_item["video_start_times"].data) video_start_time = out_item["video_start_times"].data
assert isinstance(grid_thw, torch.Tensor)
assert isinstance(second_per_grid_ts, (int, float, torch.Tensor))
assert isinstance(video_start_time, (int, float, torch.Tensor))
spt = float(second_per_grid_ts)
start = float(video_start_time)
T, H, W = map(int, grid_thw) T, H, W = map(int, grid_thw)
n_per_grid = H * W // (merge_size * merge_size) n_per_grid = H * W // (merge_size * merge_size)
# Check if this is a video_audio item # Check if this is a video_audio item
n_segs_field = out_item.get("video_audio_n_segs") n_segs_field = out_item.get("video_audio_n_segs")
n_segs_val = int(n_segs_field.data) if n_segs_field is not None else 0 if n_segs_field is None:
n_segs_val = 0
else:
n_segs_data = n_segs_field.data
assert isinstance(n_segs_data, (int, torch.Tensor))
n_segs_val = int(n_segs_data)
va_seg_lens: list[int] | None = None va_seg_lens: list[int] | None = None
if n_segs_val > 0: if n_segs_val > 0:
seg_lens_field = out_item.get("video_audio_seg_lens") seg_lens_field = out_item.get("video_audio_seg_lens")
if seg_lens_field is not None: if seg_lens_field is not None:
va_seg_lens = seg_lens_field.data[:n_segs_val].tolist() seg_lens_data = seg_lens_field.data
assert isinstance(seg_lens_data, torch.Tensor)
va_seg_lens = seg_lens_data[:n_segs_val].tolist()
full: list[int] = [video_start_id] full: list[int] = [video_start_id]
is_embed_mask: list[bool] = [False] is_embed_mask: list[bool] = [False]
@@ -1099,6 +1138,9 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
is_embed_mask.append(False) is_embed_mask.append(False)
# Audio tokens for this group # Audio tokens for this group
seg_len = va_seg_lens[g] seg_len = va_seg_lens[g]
assert audio_start_id is not None
assert audio_pad_id is not None
assert audio_end_id is not None
full.append(audio_start_id) full.append(audio_start_id)
is_embed_mask.append(False) is_embed_mask.append(False)
full.extend([audio_pad_id] * seg_len) full.extend([audio_pad_id] * seg_len)
@@ -1117,8 +1159,11 @@ class MiMoV2OmniMultiModalProcessor(BaseMultiModalProcessor[MiMoV2OmniProcessing
def get_audio_replacement(item_idx: int) -> PromptUpdateDetails: def get_audio_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["audio"][item_idx] out_item = out_mm_kwargs["audio"][item_idx]
tok_len = int(out_item["audio_token_lens"].data) audio_token_lens = out_item["audio_token_lens"].data
return [audio_pad_id] * tok_len assert isinstance(audio_token_lens, (int, torch.Tensor))
assert audio_pad_id is not None
tok_len = int(audio_token_lens)
return PromptUpdateDetails.from_seq([audio_pad_id] * tok_len)
updates: list[PromptUpdate] = [ updates: list[PromptUpdate] = [
PromptReplacement( PromptReplacement(
@@ -1166,20 +1211,24 @@ class MiMoV2OmniDummyInputsBuilder(BaseDummyInputsBuilder[MiMoV2OmniProcessingIn
target_num_frames = self.info.get_num_frames_with_most_features( target_num_frames = self.info.get_num_frames_with_most_features(
seq_len, mm_counts seq_len, mm_counts
) )
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return { return {
"image": self._get_dummy_images( "image": self._get_dummy_images(
width=target_width, width=target_width,
height=target_height, height=target_height,
num_images=num_images, num_images=num_images,
overrides=mm_options.get("image"), overrides=image_overrides,
), ),
"video": self._get_dummy_videos( "video": self._get_dummy_videos(
width=target_width, width=target_width,
height=target_height, height=target_height,
num_frames=target_num_frames, num_frames=target_num_frames,
num_videos=num_videos, num_videos=num_videos,
overrides=mm_options.get("video"), overrides=video_overrides,
), ),
} }
@@ -1235,6 +1284,7 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
) )
audio_config = getattr(config, "audio_config", None) audio_config = getattr(config, "audio_config", None)
model_path = vllm_config.model_config.model model_path = vllm_config.model_config.model
self.audio_encoder: MimoAudioEncoder | None
if audio_config is not None: if audio_config is not None:
with self._mark_tower_model(vllm_config, "audio"): with self._mark_tower_model(vllm_config, "audio"):
self.audio_encoder = MimoAudioEncoder( self.audio_encoder = MimoAudioEncoder(
@@ -1269,12 +1319,12 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
image_grid_thw=image_grid_thw, image_grid_thw=image_grid_thw,
) )
if image_embeds is not None: assert image_embeds is not None
return Qwen2_5_VLImageEmbeddingInputs( return Qwen2_5_VLImageEmbeddingInputs(
type="image_embeds", type="image_embeds",
image_embeds=image_embeds, image_embeds=image_embeds,
image_grid_thw=image_grid_thw, image_grid_thw=image_grid_thw,
) )
def _parse_and_validate_video_input( def _parse_and_validate_video_input(
self, **kwargs: object self, **kwargs: object
@@ -1295,13 +1345,13 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
second_per_grid_ts=second_per_grid_ts, second_per_grid_ts=second_per_grid_ts,
) )
if video_embeds is not None: assert video_embeds is not None
return Qwen2_5_VLVideoEmbeddingInputs( return Qwen2_5_VLVideoEmbeddingInputs(
type="video_embeds", type="video_embeds",
video_embeds=video_embeds, video_embeds=video_embeds,
video_grid_thw=video_grid_thw, video_grid_thw=video_grid_thw,
second_per_grid_ts=second_per_grid_ts, second_per_grid_ts=second_per_grid_ts,
) )
def _process_image_input( def _process_image_input(
self, image_input: Qwen2_5_VLImageInputs self, image_input: Qwen2_5_VLImageInputs
@@ -1339,19 +1389,29 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist() sizes = (grid_thw.prod(-1) // merge_size // merge_size).tolist()
return video_embeds.split(sizes) return video_embeds.split(sizes)
def _parse_and_validate_audio_input(self, **kwargs: object) -> dict | None: def _parse_and_validate_audio_input(
self, **kwargs: object
) -> MiMoV2OmniAudioInputs | None:
audio_features = kwargs.pop("audio_features", None) audio_features = kwargs.pop("audio_features", None)
audio_token_lens = kwargs.pop("audio_token_lens", None) audio_token_lens = kwargs.pop("audio_token_lens", None)
if audio_features is None: if audio_features is None:
return None return None
assert isinstance(audio_features, (torch.Tensor, list))
if isinstance(audio_features, list):
assert all(isinstance(item, torch.Tensor) for item in audio_features)
assert audio_token_lens is None or isinstance(
audio_token_lens, (torch.Tensor, list)
)
return { return {
"type": "audio", "type": "audio",
"audio_features": audio_features, "audio_features": audio_features,
"audio_token_lens": audio_token_lens, "audio_token_lens": audio_token_lens,
} }
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
mm_input_by_modality = {} self, **kwargs: object
) -> MiMoV2OmniMultiModalInputs:
mm_input_by_modality: MiMoV2OmniMultiModalInputs = {}
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
@@ -1376,7 +1436,9 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
) )
return mm_input_by_modality return mm_input_by_modality
def _process_audio_input(self, audio_input: dict) -> tuple[torch.Tensor, ...]: def _process_audio_input(
self, audio_input: MiMoV2OmniAudioInputs
) -> tuple[torch.Tensor, ...]:
mel_specs = audio_input["audio_features"] mel_specs = audio_input["audio_features"]
if self.audio_encoder is None: if self.audio_encoder is None:
return () return ()
@@ -1407,38 +1469,52 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
# Pre-process va audio: one mel spec per va video → per-video audio embeddings # Pre-process va audio: one mel spec per va video → per-video audio embeddings
# keyed by va video index (0-based among va videos only) # keyed by va video index (0-based among va videos only)
va_audio_embs_list: list[tuple[torch.Tensor, ...]] = [] va_audio_embs_list: list[torch.Tensor] = []
if va_audio_features is not None and self.audio_encoder is not None: if va_audio_features is not None and self.audio_encoder is not None:
mel_list = ( assert isinstance(va_audio_features, (torch.Tensor, list, tuple))
list(va_audio_features) mel_list: list[torch.Tensor] = []
if isinstance(va_audio_features, torch.Tensor) for mel_spec in va_audio_features:
else list(va_audio_features) assert isinstance(mel_spec, torch.Tensor)
) mel_list.append(mel_spec)
for mel_spec in mel_list: for mel_spec in mel_list:
embs, tok_lens = self.audio_encoder.get_audio_feature([mel_spec]) embs, tok_lens = self.audio_encoder.get_audio_feature([mel_spec])
# tok_lens is a list/tensor with one entry (total tokens for this mel) # tok_lens is a list/tensor with one entry (total tokens for this mel)
va_audio_embs_list.append(embs) # shape (total_tok, hidden) va_audio_embs_list.append(embs) # shape (total_tok, hidden)
va_cursor = 0 # index into va_audio_embs_list va_cursor = 0 # index into va_audio_embs_list
assert video_audio_n_segs is None or isinstance(
video_audio_n_segs, torch.Tensor
)
assert video_audio_seg_lens is None or isinstance(
video_audio_seg_lens, torch.Tensor
)
# NOTE: Iterate in dict insertion order to preserve token sequence order. # NOTE: Iterate in dict insertion order to preserve token sequence order.
for modality in mm_input_by_modality: for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image": if modality == "image":
multimodal_embeddings.extend( image_input = mm_input_by_modality["image"]
self._process_image_input(multimodal_input) assert isinstance(
image_input,
(Qwen2_5_VLImagePixelInputs, Qwen2_5_VLImageEmbeddingInputs),
) )
multimodal_embeddings.extend(self._process_image_input(image_input))
elif modality == "video": elif modality == "video":
video_embs_tuple = self._process_video_input(multimodal_input) video_input = mm_input_by_modality["video"]
assert isinstance(
video_input,
(Qwen2_5_VLVideoPixelInputs, Qwen2_5_VLVideoEmbeddingInputs),
)
video_embs_tuple = self._process_video_input(video_input)
if video_audio_n_segs is None: if video_audio_n_segs is None:
multimodal_embeddings.extend(video_embs_tuple) multimodal_embeddings.extend(video_embs_tuple)
else: else:
grid_thw = multimodal_input["video_grid_thw"] grid_thw = video_input["video_grid_thw"]
for i, vid_embs in enumerate(video_embs_tuple): for i, vid_embs in enumerate(video_embs_tuple):
n_segs = int(video_audio_n_segs[i]) n_segs = int(video_audio_n_segs[i])
if n_segs == 0 or not va_audio_embs_list: if n_segs == 0 or not va_audio_embs_list:
multimodal_embeddings.append(vid_embs) multimodal_embeddings.append(vid_embs)
else: else:
assert video_audio_seg_lens is not None
T = int(grid_thw[i][0]) T = int(grid_thw[i][0])
n_per_grid = vid_embs.shape[0] // T n_per_grid = vid_embs.shape[0] // T
frames = list(vid_embs.split(n_per_grid, dim=0)) frames = list(vid_embs.split(n_per_grid, dim=0))
@@ -1459,9 +1535,9 @@ class MiMoV2OmniForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsQ
) )
multimodal_embeddings.append(group_audio_embs[g]) multimodal_embeddings.append(group_audio_embs[g])
elif modality == "audio": elif modality == "audio":
multimodal_embeddings.extend( audio_input = mm_input_by_modality["audio"]
self._process_audio_input(multimodal_input) assert audio_input is not None
) multimodal_embeddings.extend(self._process_audio_input(audio_input))
return tuple(multimodal_embeddings) return tuple(multimodal_embeddings)
def forward( def forward(
+1
View File
@@ -456,6 +456,7 @@ class MiniCPMModel(nn.Module, EagleModelMixin):
hidden_states = self.embed_input_ids(input_ids) hidden_states = self.embed_input_ids(input_ids)
residual = None residual = None
else: else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"] hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"] residual = intermediate_tensors["residual"]
+18 -3
View File
@@ -26,6 +26,7 @@
import math import math
from collections.abc import Iterable from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch import torch
from torch import nn from torch import nn
@@ -55,6 +56,14 @@ from .utils import (
process_eagle_weight, process_eagle_weight,
) )
if TYPE_CHECKING:
class _EagleMiniCPMSupportsPP:
pass
else:
_EagleMiniCPMSupportsPP = SupportsPP
class EagleMiniCPMDecoderLayer(nn.Module): class EagleMiniCPMDecoderLayer(nn.Module):
def __init__( def __init__(
@@ -149,7 +158,9 @@ class EagleMiniCPMModel(nn.Module):
): ):
super().__init__() super().__init__()
config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
cache_config = vllm_config.cache_config cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
@@ -291,7 +302,9 @@ class EagleMiniCPMModel(nn.Module):
return loaded_params return loaded_params
class EagleMiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle): class EagleMiniCPMForCausalLM(
nn.Module, SupportsLoRA, _EagleMiniCPMSupportsPP, SupportsEagle
):
packed_modules_mapping = { packed_modules_mapping = {
"qkv_proj": [ "qkv_proj": [
"q_proj", "q_proj",
@@ -312,7 +325,9 @@ class EagleMiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__() super().__init__()
config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
cache_config = vllm_config.cache_config cache_config = vllm_config.cache_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
+61 -17
View File
@@ -40,7 +40,7 @@ from transformers.models.whisper.modeling_whisper import (
) )
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.inputs import ModalityData, MultiModalDataDict from vllm.inputs import ModalityData, MultiModalDataDict
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargsItems
from vllm.multimodal.inputs import ( from vllm.multimodal.inputs import (
@@ -66,8 +66,12 @@ from .minicpmv import (
_MAX_FRAMES_PER_VIDEO, _MAX_FRAMES_PER_VIDEO,
MiniCPMV2_6, MiniCPMV2_6,
MiniCPMV4_5, MiniCPMV4_5,
MiniCPMVBaseModel,
MiniCPMVDummyInputsBuilder, MiniCPMVDummyInputsBuilder,
MiniCPMVImageEmbeddingInputs,
MiniCPMVImagePixelInputs,
MiniCPMVMultiModalDataParser, MiniCPMVMultiModalDataParser,
MiniCPMVMultiModalInputs,
MiniCPMVMultiModalProcessor, MiniCPMVMultiModalProcessor,
MiniCPMVProcessingInfo, MiniCPMVProcessingInfo,
_minicpmv_field_config, _minicpmv_field_config,
@@ -181,6 +185,10 @@ MiniCPMOAudioInputs: TypeAlias = (
) )
class MiniCPMOMultiModalInputs(MiniCPMVMultiModalInputs, total=False):
audios: MiniCPMOAudioInputs | None
def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]): def _minicpmo_field_config(hf_inputs: Mapping[str, torch.Tensor]):
audio_features = hf_inputs.get("audio_features") audio_features = hf_inputs.get("audio_features")
audio_feature_lens = hf_inputs.get("audio_feature_lens") audio_feature_lens = hf_inputs.get("audio_feature_lens")
@@ -405,6 +413,7 @@ class MiniCPMODummyInputsBuilder(MiniCPMVDummyInputsBuilder[MiniCPMOProcessingIn
) )
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
audio_mm_data = { audio_mm_data = {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
@@ -469,7 +478,8 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
if isinstance(lens, torch.Tensor): if isinstance(lens, torch.Tensor):
flat_feature_lens.extend(lens.flatten().tolist()) flat_feature_lens.extend(lens.flatten().tolist())
else: else:
flat_feature_lens.append(int(lens)) assert isinstance(lens, int)
flat_feature_lens.append(lens)
unpadded_audio_features = [ unpadded_audio_features = [
feat[:, :length] feat[:, :length]
for feat, length in zip( for feat, length in zip(
@@ -515,6 +525,9 @@ class MiniCPMOMultiModalProcessor(MiniCPMVMultiModalProcessor[MiniCPMOProcessing
audios = mm_items.get_items( audios = mm_items.get_items(
"audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems) "audio", (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
) )
assert isinstance(
audios, (MiniCPMOAudioEmbeddingItems, AudioProcessorItems)
)
if isinstance(audios, MiniCPMOAudioEmbeddingItems): if isinstance(audios, MiniCPMOAudioEmbeddingItems):
single_audio_embeds = audios.get(item_idx)["audio_embeds"] single_audio_embeds = audios.get(item_idx)["audio_embeds"]
@@ -652,7 +665,7 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
hidden_states, p=self.dropout, training=self.training hidden_states, p=self.dropout, training=self.training
) )
encoder_states = () encoder_states: tuple[torch.Tensor, ...] = ()
for idx, encoder_layer in enumerate(self.layers): for idx, encoder_layer in enumerate(self.layers):
encoder_states = encoder_states + (hidden_states,) encoder_states = encoder_states + (hidden_states,)
@@ -682,7 +695,13 @@ class MiniCPMWhisperEncoder(WhisperEncoder):
) )
class MiniCPMOBaseModel: if TYPE_CHECKING:
_MiniCPMOBaseModelBase = MiniCPMVBaseModel
else:
_MiniCPMOBaseModelBase = object
class MiniCPMOBaseModel(_MiniCPMOBaseModelBase):
"""Base mixin class for MiniCPM-O models with audio support.""" """Base mixin class for MiniCPM-O models with audio support."""
# Unlike the vision-only MiniCPM-V models, audio weights are loaded here. # Unlike the vision-only MiniCPM-V models, audio weights are loaded here.
@@ -897,8 +916,12 @@ class MiniCPMOBaseModel:
audio_feature_lens=audio_feature_lens, audio_feature_lens=audio_feature_lens,
) )
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = super()._parse_and_validate_multimodal_inputs(**kwargs) self, **kwargs: object
) -> MiniCPMOMultiModalInputs:
modalities = MiniCPMOMultiModalInputs(
**super()._parse_and_validate_multimodal_inputs(**kwargs)
)
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
@@ -918,16 +941,33 @@ class MiniCPMOBaseModel:
if audio_input["type"] == "audio_embeds": if audio_input["type"] == "audio_embeds":
return audio_input["audio_embeds"] return audio_input["audio_embeds"]
assert isinstance(audio_input, MiniCPMOAudioFeatureInputs)
return self.get_audio_hidden_states(audio_input) return self.get_audio_hidden_states(audio_input)
def _process_multimodal_inputs(self, modalities: dict): def _process_multimodal_inputs(self, modalities: Mapping[str, object]):
multimodal_embeddings = super()._process_multimodal_inputs(modalities) base_modalities: MiniCPMVMultiModalInputs = {}
image_input = modalities.get("images")
video_input = modalities.get("videos")
assert image_input is None or isinstance(
image_input, (MiniCPMVImagePixelInputs, MiniCPMVImageEmbeddingInputs)
)
assert video_input is None or isinstance(
video_input, (MiniCPMVImagePixelInputs, MiniCPMVImageEmbeddingInputs)
)
if "images" in modalities:
base_modalities["images"] = image_input
if "videos" in modalities:
base_modalities["videos"] = video_input
multimodal_embeddings = super()._process_multimodal_inputs(base_modalities)
for modality in modalities: audio_input = modalities.get("audios")
if modality == "audios": if audio_input is not None:
audio_input = modalities["audios"] assert isinstance(
audio_embeddings = self._process_audio_input(audio_input) audio_input,
multimodal_embeddings += tuple(audio_embeddings) (MiniCPMOAudioFeatureInputs, MiniCPMOAudioEmbeddingInputs),
)
audio_embeddings = self._process_audio_input(audio_input)
multimodal_embeddings += tuple(audio_embeddings)
return multimodal_embeddings return multimodal_embeddings
@@ -983,7 +1023,7 @@ class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
try: try:
version_str = str(config.version) version_str = str(config.version)
version_parts = version_str.split(".") version_parts = version_str.split(".")
version = tuple(int(x) for x in version_parts[:2]) version_values = tuple(int(x) for x in version_parts[:2])
except (ValueError, TypeError) as e: except (ValueError, TypeError) as e:
raise ValueError( raise ValueError(
f"Invalid model version format in config: {config.version}. " f"Invalid model version format in config: {config.version}. "
@@ -991,17 +1031,21 @@ class MiniCPMO(MiniCPMOBaseModel, MiniCPMV2_6):
) from e ) from e
else: else:
# Default to 2.6 for backward compatibility # Default to 2.6 for backward compatibility
version = (2, 6) version_values = (2, 6)
# Dispatch class based on version # Dispatch class based on version
instance_cls = _MINICPMO_SUPPORT_VERSION.get(version) if len(version_values) == 2:
version_key = (version_values[0], version_values[1])
instance_cls = _MINICPMO_SUPPORT_VERSION.get(version_key)
else:
instance_cls = None
if instance_cls is None: if instance_cls is None:
supported_versions = ", ".join( supported_versions = ", ".join(
[f"{v[0]}.{v[1]}" for v in sorted(_MINICPMO_SUPPORT_VERSION.keys())] [f"{v[0]}.{v[1]}" for v in sorted(_MINICPMO_SUPPORT_VERSION.keys())]
) )
raise ValueError( raise ValueError(
f"Currently, MiniCPMO only supports versions " f"Currently, MiniCPMO only supports versions "
f"{supported_versions}. Got version: {version}" f"{supported_versions}. Got version: {version_values}"
) )
return instance_cls(vllm_config=vllm_config, prefix=prefix) return instance_cls(vllm_config=vllm_config, prefix=prefix)
+43 -13
View File
@@ -41,7 +41,7 @@ from transformers.dynamic_module_utils import (
get_class_from_dynamic_module, get_class_from_dynamic_module,
resolve_trust_remote_code, resolve_trust_remote_code,
) )
from typing_extensions import TypeVar from typing_extensions import TypedDict, TypeVar
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import ( from vllm.config.multimodal import (
@@ -97,7 +97,7 @@ from vllm.transformers_utils.processor import (
cached_get_image_processor, cached_get_image_processor,
) )
from vllm.transformers_utils.utils import convert_model_repo_to_path from vllm.transformers_utils.utils import convert_model_repo_to_path
from vllm.utils.collection_utils import flatten_2d_lists from vllm.utils.collection_utils import flatten_2d_lists, is_list_of
from vllm.utils.gpu_sync_debug import gpu_sync_allowed from vllm.utils.gpu_sync_debug import gpu_sync_allowed
from vllm.utils.tensor_schema import TensorSchema, TensorShape from vllm.utils.tensor_schema import TensorSchema, TensorShape
from vllm.utils.torch_utils import set_default_torch_dtype from vllm.utils.torch_utils import set_default_torch_dtype
@@ -386,6 +386,7 @@ class Resampler4_5(Resampler2_5):
for i in range(bs): for i in range(bs):
tgt_h, tgt_w = tgt_sizes[i] tgt_h, tgt_w = tgt_sizes[i]
if temporal_pos_emb: if temporal_pos_emb:
assert temporal_ids_flatten is not None
if temporal_ids_flatten[i] == -1: if temporal_ids_flatten[i] == -1:
pos_embed_temporal.append( pos_embed_temporal.append(
torch.zeros(self.embed_dim, dtype=dtype, device=device) torch.zeros(self.embed_dim, dtype=dtype, device=device)
@@ -549,6 +550,11 @@ class MiniCPMVVideoEmbeddingItems(DictEmbeddingItems):
return len(self.get(index)["video_image_sizes"]) return len(self.get(index)["video_image_sizes"])
class MiniCPMVMultiModalInputs(TypedDict, total=False):
images: MiniCPMVImageInputs | None
videos: MiniCPMVImageInputs | None
class MiniCPMVMultiModalDataParser(MultiModalDataParser): class MiniCPMVMultiModalDataParser(MultiModalDataParser):
# The per-item sizes are what size the placeholder range. # The per-item sizes are what size the placeholder range.
embedding_fields = { embedding_fields = {
@@ -833,6 +839,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
image_overrides = mm_options.get("image") image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video") video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
# Convert video overrides to image overrides for per-frame image generation, # Convert video overrides to image overrides for per-frame image generation,
# and apply num_frames override to num_video_frames. # and apply num_frames override to num_video_frames.
@@ -842,6 +849,7 @@ class MiniCPMVDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
num_video_frames = min(num_video_frames, video_overrides.num_frames) num_video_frames = min(num_video_frames, video_overrides.num_frames)
if video_overrides.width or video_overrides.height: if video_overrides.width or video_overrides.height:
video_frame_overrides = ImageDummyOptions( video_frame_overrides = ImageDummyOptions(
count=video_overrides.count,
width=video_overrides.width, width=video_overrides.width,
height=video_overrides.height, height=video_overrides.height,
) )
@@ -1081,6 +1089,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
images = mm_items.get_items( images = mm_items.get_items(
"image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems) "image", (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
) )
assert isinstance(
images, (MiniCPMVImageEmbeddingItems, ImageProcessorItems)
)
image_size = images.get_image_size(item_idx) image_size = images.get_image_size(item_idx)
@@ -1097,6 +1108,9 @@ class MiniCPMVMultiModalProcessor(BaseMultiModalProcessor[_I]):
videos = mm_items.get_items( videos = mm_items.get_items(
"video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems) "video", (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
) )
assert isinstance(
videos, (MiniCPMVVideoEmbeddingItems, VideoProcessorItems)
)
frame_size = videos.get_frame_size(item_idx) frame_size = videos.get_frame_size(item_idx)
num_frames = videos.get_num_frames(item_idx) num_frames = videos.get_num_frames(item_idx)
@@ -1199,6 +1213,7 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
super().__init__() super().__init__()
@@ -1261,10 +1276,15 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
image_embeds=image_embeds, image_embeds=image_embeds,
) )
assert is_list_of(pixel_values, list, check="all")
pixel_values_list: list[list[torch.Tensor]] = []
for pixel_value in pixel_values:
assert is_list_of(pixel_value, torch.Tensor, check="all")
pixel_values_list.append(pixel_value)
tgt_sizes = kwargs.pop("tgt_sizes") tgt_sizes = kwargs.pop("tgt_sizes")
num_slices_flat = torch.tensor([len(ps) for ps in pixel_values]) num_slices_flat = torch.tensor([len(ps) for ps in pixel_values_list])
pixel_values_flat = flatten_bn(pixel_values) pixel_values_flat = flatten_2d_lists(pixel_values_list)
tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True) tgt_sizes_flat = flatten_bn(tgt_sizes, concat=True)
return MiniCPMVImagePixelInputs( return MiniCPMVImagePixelInputs(
@@ -1274,8 +1294,11 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
num_slices=num_slices_flat, num_slices=num_slices_flat,
) )
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = {} self, **kwargs: object
) -> MiniCPMVMultiModalInputs:
kwargs.pop("modality", None)
modalities: MiniCPMVMultiModalInputs = {}
# Preserve the order of modalities if there are multiple of them # Preserve the order of modalities if there are multiple of them
# from the order of kwargs. # from the order of kwargs.
@@ -1304,12 +1327,13 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
if image_input["type"] == "image_embeds": if image_input["type"] == "image_embeds":
return image_input["image_embeds"] return image_input["image_embeds"]
assert isinstance(image_input, MiniCPMVImagePixelInputs)
image_features_flat = self.get_vision_hidden_states(image_input) image_features_flat = self.get_vision_hidden_states(image_input)
num_slices = image_input["num_slices"] num_slices = image_input["num_slices"]
return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())] return [e.flatten(0, 1) for e in image_features_flat.split(num_slices.tolist())]
def _process_multimodal_inputs(self, modalities: dict): def _process_multimodal_inputs(self, modalities: MiniCPMVMultiModalInputs):
# The result multimodal_embeddings is tuple of tensors, with each # The result multimodal_embeddings is tuple of tensors, with each
# tensor corresponding to a multimodal data item (image or video). # tensor corresponding to a multimodal data item (image or video).
multimodal_embeddings: tuple[torch.Tensor, ...] = () multimodal_embeddings: tuple[torch.Tensor, ...] = ()
@@ -1319,10 +1343,12 @@ class MiniCPMVBaseModel(nn.Module, SupportsMultiModal, SupportsPP):
for modality in modalities: for modality in modalities:
if modality == "images": if modality == "images":
image_input = modalities["images"] image_input = modalities["images"]
assert image_input is not None
image_embeddings = self._process_vision_input(image_input) image_embeddings = self._process_vision_input(image_input)
multimodal_embeddings += tuple(image_embeddings) multimodal_embeddings += tuple(image_embeddings)
if modality == "videos": if modality == "videos":
video_input = modalities["videos"] video_input = modalities["videos"]
assert video_input is not None
video_embeddings = self._process_vision_input(video_input) video_embeddings = self._process_vision_input(video_input)
multimodal_embeddings += tuple(video_embeddings) multimodal_embeddings += tuple(video_embeddings)
@@ -1915,23 +1941,27 @@ class MiniCPMV(MiniCPMVBaseModel, SupportsMultiModal, SupportsLoRA):
def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""): def __new__(cls, *, vllm_config: VllmConfig, prefix: str = ""):
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
version_values: tuple[int, ...]
if not hasattr(config, "version"): if not hasattr(config, "version"):
if config.hidden_size == 2304 and config.query_num == 64: if config.hidden_size == 2304 and config.query_num == 64:
version = (2, 0) version_values = (2, 0)
else: else:
version = (2, 5) version_values = (2, 5)
else: else:
version = str(config.version).split(".") version_values = tuple(int(x) for x in str(config.version).split("."))
version = tuple([int(x) for x in version])
# Dispatch class based on version # Dispatch class based on version
instance_cls = _SUPPORT_VERSION.get(version) if len(version_values) == 2:
version_key = (version_values[0], version_values[1])
instance_cls = _SUPPORT_VERSION.get(version_key)
else:
instance_cls = None
if instance_cls is None: if instance_cls is None:
supported_versions = ", ".join( supported_versions = ", ".join(
[f"{v[0]}.{v[1]}" for v in sorted(_SUPPORT_VERSION.keys())] [f"{v[0]}.{v[1]}" for v in sorted(_SUPPORT_VERSION.keys())]
) )
raise ValueError( raise ValueError(
f"Currently, MiniCPMV only supports versions " f"Currently, MiniCPMV only supports versions "
f"{supported_versions}. Got version: {version}" f"{supported_versions}. Got version: {version_values}"
) )
# quant_config references base class members, # quant_config references base class members,
+9 -2
View File
@@ -98,7 +98,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
ds = mm_kwargs.get("downsample_mode") ds = mm_kwargs.get("downsample_mode")
if ds is not None: if ds is not None:
return str(ds) return str(ds)
return self.info._get_downsample_mode() info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
return info._get_downsample_mode()
def get_image_prompt_texts( def get_image_prompt_texts(
self, self,
@@ -106,7 +108,9 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
image_idx: int = 0, image_idx: int = 0,
downsample_mode: str | None = None, downsample_mode: str | None = None,
) -> str: ) -> str:
return self.info.get_slice_image_placeholder( info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
return info.get_slice_image_placeholder(
image_size, image_size,
image_idx=image_idx, image_idx=image_idx,
downsample_mode=downsample_mode, downsample_mode=downsample_mode,
@@ -128,6 +132,7 @@ class MiniCPMV4_6MultiModalProcessor(MiniCPMVMultiModalProcessor):
# which one is used. Using image_token for video silently produces # which one is used. Using image_token for video silently produces
# garbage descriptions. # garbage descriptions.
info = self.info info = self.info
assert isinstance(info, MiniCPMV4_6ProcessingInfo)
grids, source_tokens, patch_tokens = info._compute_visual_tokens( grids, source_tokens, patch_tokens = info._compute_visual_tokens(
image_size, image_size,
max_slice_nums=info.get_video_max_slice_num(), max_slice_nums=info.get_video_max_slice_num(),
@@ -1016,6 +1021,7 @@ class MiniCPMV4_6ForConditionalGeneration(
config: MiniCPMV4_6Config = vllm_config.model_config.hf_config config: MiniCPMV4_6Config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
self.config = config self.config = config
self.multimodal_config = multimodal_config self.multimodal_config = multimodal_config
@@ -1078,6 +1084,7 @@ class MiniCPMV4_6ForConditionalGeneration(
image_embeds=image_embeds, image_embeds=image_embeds,
) )
assert isinstance(pixel_values, torch.Tensor | list)
tgt_sizes = kwargs.pop("tgt_sizes") tgt_sizes = kwargs.pop("tgt_sizes")
num_slices_flat = torch.tensor([len(ps) for ps in pixel_values]) num_slices_flat = torch.tensor([len(ps) for ps in pixel_values])
pixel_values_flat = flatten_bn(pixel_values) pixel_values_flat = flatten_bn(pixel_values)
+8 -2
View File
@@ -221,15 +221,21 @@ class MistralModel(LlamaModel):
intermediate_tensors: IntermediateTensors | None, intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None, inputs_embeds: torch.Tensor | None = None,
t_cond: torch.Tensor | None = None, t_cond: torch.Tensor | None = None,
**extra_layer_kwargs: object,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]: ) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
return super().forward( return super().forward(
input_ids, positions, intermediate_tensors, inputs_embeds, t_cond=t_cond input_ids,
positions,
intermediate_tensors,
inputs_embeds,
t_cond=t_cond,
**extra_layer_kwargs,
) )
class MistralForCausalLM(LlamaForCausalLM): class MistralForCausalLM(LlamaForCausalLM):
# Mistral: We don't support LoRA on the embedding layers. # Mistral: We don't support LoRA on the embedding layers.
embedding_modules: dict[str, str] = {} embedding_modules = {}
# Mistral/Llama models can also be loaded with --load-format mistral # Mistral/Llama models can also be loaded with --load-format mistral
# from consolidated.safetensors checkpoints # from consolidated.safetensors checkpoints
+1
View File
@@ -196,6 +196,7 @@ class Mistral3HFEncoderInfo(PixtralHFEncoderInfo):
image_height = math.floor(image_height / ratio) image_height = math.floor(image_height / ratio)
patch_size = self.vision_config.patch_size patch_size = self.vision_config.patch_size
assert isinstance(self.hf_config, Mistral3Config)
spatial_merge_size = self.hf_config.spatial_merge_size spatial_merge_size = self.hf_config.spatial_merge_size
# The HF processor rounds each dimension up to the vision patch size # The HF processor rounds each dimension up to the vision patch size
+25 -4
View File
@@ -2,6 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable from collections.abc import Iterable
from typing import TYPE_CHECKING
import torch import torch
import torch.nn as nn import torch.nn as nn
@@ -33,6 +34,20 @@ from vllm.model_executor.models.utils import (
logger = init_logger(__name__) logger = init_logger(__name__)
if TYPE_CHECKING:
class _EagleMistralModelBase(nn.Module):
pass
class _EagleMistralForCausalLMBase(nn.Module):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
raise NotImplementedError
else:
_EagleMistralModelBase = MistralModel
_EagleMistralForCausalLMBase = MistralForCausalLM
class EagleMistralDecoderLayer(MistralDecoderLayer): class EagleMistralDecoderLayer(MistralDecoderLayer):
def __init__( def __init__(
self, self,
@@ -47,7 +62,7 @@ class EagleMistralDecoderLayer(MistralDecoderLayer):
@support_torch_compile @support_torch_compile
class EagleMistralModel(MistralModel): class EagleMistralModel(_EagleMistralModelBase):
def __init__( def __init__(
self, self,
*, *,
@@ -58,7 +73,9 @@ class EagleMistralModel(MistralModel):
# Bypass MistralModel.__init__ to avoid creating duplicate attention # Bypass MistralModel.__init__ to avoid creating duplicate attention
# layer entries in the global context. # layer entries in the global context.
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
self.vocab_size = self.config.vocab_size self.vocab_size = self.config.vocab_size
# Get drafter's quantization config # Get drafter's quantization config
self.quant_config = get_draft_quant_config(vllm_config) self.quant_config = get_draft_quant_config(vllm_config)
@@ -112,7 +129,9 @@ class EagleMistralModel(MistralModel):
return hidden_states, hidden_states return hidden_states, hidden_states
class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings): class EagleMistralForCausalLM(
_EagleMistralForCausalLMBase, SupportsMultiModalEmbeddings
):
mistral_mapping = MistralForCausalLM.mistral_mapping | { mistral_mapping = MistralForCausalLM.mistral_mapping | {
"eagle_linear": "model.fc", "eagle_linear": "model.fc",
} }
@@ -121,7 +140,9 @@ class EagleMistralForCausalLM(MistralForCausalLM, SupportsMultiModalEmbeddings):
# Bypass MistralForCausalLM.__init__ to use the draft model config # Bypass MistralForCausalLM.__init__ to use the draft model config
# and to avoid creating an lm_head. # and to avoid creating an lm_head.
nn.Module.__init__(self) nn.Module.__init__(self)
self.config = vllm_config.speculative_config.draft_model_config.hf_config speculative_config = vllm_config.speculative_config
assert speculative_config is not None
self.config = speculative_config.draft_model_config.hf_config
target_layer_num = vllm_config.model_config.get_num_layers( target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config vllm_config.parallel_config
) )
@@ -4,6 +4,7 @@
import copy import copy
from collections.abc import Iterable from collections.abc import Iterable
from functools import partial from functools import partial
from typing import TYPE_CHECKING
import regex import regex
import torch import torch
@@ -28,6 +29,15 @@ from .utils import WeightsMapper, make_empty_intermediate_tensors_factory, maybe
logger = init_logger(__name__) logger = init_logger(__name__)
if TYPE_CHECKING:
class _EagleMistralLarge3ForCausalLMBase(nn.Module):
pass
else:
_EagleMistralLarge3ForCausalLMBase = MistralLarge3ForCausalLM
@support_torch_compile @support_torch_compile
class EagleMistralLarge3Model(DeepseekV2Model): class EagleMistralLarge3Model(DeepseekV2Model):
def __init__( def __init__(
@@ -108,7 +118,7 @@ class EagleMistralLarge3Model(DeepseekV2Model):
class EagleMistralLarge3ForCausalLM( class EagleMistralLarge3ForCausalLM(
MistralLarge3ForCausalLM, SupportsMultiModalEmbeddings _EagleMistralLarge3ForCausalLMBase, SupportsMultiModalEmbeddings
): ):
hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper( hf_to_vllm_mapper = MistralLarge3ForCausalLM.hf_to_vllm_mapper | WeightsMapper(
orig_to_new_regex={ orig_to_new_regex={
@@ -119,13 +129,15 @@ class EagleMistralLarge3ForCausalLM(
) )
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""): def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
target_layer_num = vllm_config.model_config.get_num_layers( target_layer_num = vllm_config.model_config.get_num_layers(
vllm_config.parallel_config vllm_config.parallel_config
) )
vllm_config.model_config = vllm_config.speculative_config.draft_model_config vllm_config.model_config = speculative_config.draft_model_config
# draft model quantization config may differ from target model # draft model quantization config may differ from target model
self.quant_config = VllmConfig.get_quantization_config( self.quant_config = VllmConfig.get_quantization_config(
vllm_config.speculative_config.draft_model_config, vllm_config.load_config speculative_config.draft_model_config, vllm_config.load_config
) )
vllm_config.quant_config = self.quant_config vllm_config.quant_config = self.quant_config
self.model_cls = partial( self.model_cls = partial(
+10 -7
View File
@@ -620,14 +620,16 @@ class Mllama4MultiModalProcessor(BaseMultiModalProcessor[Mllama4ProcessingInfo])
max_num_chunks=self.info.get_max_num_tiles(), max_num_chunks=self.info.get_max_num_tiles(),
patch_size=SizeDict(height=tile_size, width=tile_size), patch_size=SizeDict(height=tile_size, width=tile_size),
) )
best_fit_sizes = [ best_fit_sizes = []
get_best_fit( for image in parsed_images:
(image.size[1], image.size[0]), assert image is not None
torch.tensor(possible_resolutions), best_fit_sizes.append(
resize_to_max_canvas=image_processor.resize_to_max_canvas, get_best_fit(
(image.size[1], image.size[0]),
torch.tensor(possible_resolutions),
resize_to_max_canvas=image_processor.resize_to_max_canvas,
)
) )
for image in parsed_images
]
# TODO tile height/width do not necessarily need to match # TODO tile height/width do not necessarily need to match
aspect_ratios = [ aspect_ratios = [
(image_size[0] // tile_size, image_size[1] // tile_size) (image_size[0] // tile_size, image_size[1] // tile_size)
@@ -760,6 +762,7 @@ class Llama4ForConditionalGeneration(
config = vllm_config.model_config.hf_config config = vllm_config.model_config.hf_config
quant_config = vllm_config.quant_config quant_config = vllm_config.quant_config
multimodal_config = vllm_config.model_config.multimodal_config multimodal_config = vllm_config.model_config.multimodal_config
assert multimodal_config is not None
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data" self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.vllm_config = vllm_config self.vllm_config = vllm_config
+4 -4
View File
@@ -17,10 +17,10 @@ class MultiModelKeys:
@staticmethod @staticmethod
def from_string_field( def from_string_field(
language_model: str | list[str] = None, language_model: str | list[str] | None = None,
connector: str | list[str] = None, connector: str | list[str] | None = None,
tower_model: str | list[str] = None, tower_model: str | list[str] | None = None,
generator: str | list[str] = None, generator: str | list[str] | None = None,
**kwargs, **kwargs,
) -> "MultiModelKeys": ) -> "MultiModelKeys":
def to_list(value): def to_list(value):
+7 -4
View File
@@ -332,7 +332,7 @@ class VisionTransformer(nn.Module):
): ):
super().__init__() super().__init__()
scale = config.image_emb_dim**-0.5 scale = config.image_emb_dim**-0.5
self.patch_num = config.image_num_patch self.patch_num: tuple[int, int] = config.image_num_patch
self.class_embedding = nn.Parameter(torch.randn(config.image_emb_dim) * scale) self.class_embedding = nn.Parameter(torch.randn(config.image_emb_dim) * scale)
self.num_prefix_tokens: int = NUM_PREFIX_TOKENS self.num_prefix_tokens: int = NUM_PREFIX_TOKENS
self.positional_embedding = nn.Parameter( self.positional_embedding = nn.Parameter(
@@ -349,7 +349,7 @@ class VisionTransformer(nn.Module):
config, quant_config, prefix=f"{prefix}.transformer" config, quant_config, prefix=f"{prefix}.transformer"
) )
def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor: def add_pos_emb(self, x: torch.Tensor, patch_num: tuple[int, int]) -> torch.Tensor:
cls_emb = self.positional_embedding[0:1] cls_emb = self.positional_embedding[0:1]
pos_emb = self.positional_embedding[1:] pos_emb = self.positional_embedding[1:]
@@ -380,7 +380,7 @@ class VisionTransformer(nn.Module):
return x return x
def forward( def forward(
self, x: torch.Tensor, patch_num: int | None = None self, x: torch.Tensor, patch_num: tuple[int, int] | None = None
) -> list[torch.Tensor]: ) -> list[torch.Tensor]:
""" """
: param x: (batch_size, num_patch, n_pixels) : param x: (batch_size, num_patch, n_pixels)
@@ -484,12 +484,15 @@ class MolmoAttention(nn.Module):
def _apply_qk_norm( def _apply_qk_norm(
self, q: torch.Tensor, k: torch.Tensor self, q: torch.Tensor, k: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
assert self.q_norm is not None
assert self.k_norm is not None
if self.tp_size > 1: if self.tp_size > 1:
q = tensor_model_parallel_all_gather(q.contiguous()) q = tensor_model_parallel_all_gather(q.contiguous())
k = tensor_model_parallel_all_gather(k.contiguous()) k = tensor_model_parallel_all_gather(k.contiguous())
q = self.q_norm(q) q = self.q_norm(q)
k = self.k_norm(k) k = self.k_norm(k)
if self.tp_size > 1: if self.tp_size > 1:
assert self.tp_rank is not None
splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size) splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size)
q = splitter(q)[self.tp_rank] q = splitter(q)[self.tp_rank]
k = splitter(k)[self.tp_rank] k = splitter(k)[self.tp_rank]
@@ -1345,7 +1348,7 @@ class MolmoForCausalLM(
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model") vllm_config=vllm_config, prefix=maybe_prefix(prefix, "model")
) )
self.img_patch_id = None self.img_patch_id: int | None = None
self.lm_head = ParallelLMHead( self.lm_head = ParallelLMHead(
config.embedding_size or config.vocab_size, config.embedding_size or config.vocab_size,
+41 -9
View File
@@ -22,6 +22,7 @@ from transformers import (
) )
from transformers.image_utils import ImageInput from transformers.image_utils import ImageInput
from transformers.video_utils import VideoMetadata from transformers.video_utils import VideoMetadata
from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig from vllm.config import CacheConfig, VllmConfig
@@ -157,6 +158,11 @@ class Molmo2VideoInputs(TensorSchema):
num_video_tokens: Annotated[torch.Tensor, TensorShape("nv")] num_video_tokens: Annotated[torch.Tensor, TensorShape("nv")]
class Molmo2MultiModalInputs(TypedDict, total=False):
images: Molmo2ImageInputs | None
videos: Molmo2VideoInputs | None
@dataclass @dataclass
class VitConfig: class VitConfig:
"""Config for a vision transformer""" """Config for a vision transformer"""
@@ -474,7 +480,7 @@ class Molmo2VisionTransformer(nn.Module):
super().__init__() super().__init__()
scale = config.hidden_size**-0.5 scale = config.hidden_size**-0.5
self.num_prefix_tokens: int = 0 # no class embeddings self.num_prefix_tokens: int = 0 # no class embeddings
self.patch_num = config.image_num_patch self.patch_num: tuple[int, int] = config.image_num_patch
self.positional_embedding = nn.Parameter( self.positional_embedding = nn.Parameter(
torch.randn(config.image_num_pos, config.hidden_size) * scale, torch.randn(config.image_num_pos, config.hidden_size) * scale,
) )
@@ -490,7 +496,7 @@ class Molmo2VisionTransformer(nn.Module):
prefix=maybe_prefix(prefix, "transformer"), prefix=maybe_prefix(prefix, "transformer"),
) )
def add_pos_emb(self, x: torch.Tensor, patch_num: int) -> torch.Tensor: def add_pos_emb(self, x: torch.Tensor, patch_num: tuple[int, int]) -> torch.Tensor:
pos_emb = self.positional_embedding pos_emb = self.positional_embedding
pos_emb = pos_emb.reshape( pos_emb = pos_emb.reshape(
@@ -522,7 +528,7 @@ class Molmo2VisionTransformer(nn.Module):
def forward( def forward(
self, self,
x: torch.Tensor, x: torch.Tensor,
patch_num: int | None = None, patch_num: tuple[int, int] | None = None,
) -> list[torch.Tensor]: ) -> list[torch.Tensor]:
""" """
: param x: (batch_size, num_patch, n_pixels) : param x: (batch_size, num_patch, n_pixels)
@@ -652,6 +658,7 @@ class ImagePoolingAttention(nn.Module):
if self.use_pytorch_sdpa: if self.use_pytorch_sdpa:
output = self.forward_sdpa(xq, xk, xv, attn_mask) output = self.forward_sdpa(xq, xk, xv, attn_mask)
else: else:
assert self.attn is not None
output = self.attn(xq, xk, xv) output = self.attn(xq, xk, xv)
output, _ = self.o_proj(output) output, _ = self.o_proj(output)
@@ -954,12 +961,15 @@ class Molmo2Attention(nn.Module):
q: torch.Tensor, q: torch.Tensor,
k: torch.Tensor, k: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor]: ) -> tuple[torch.Tensor, torch.Tensor]:
assert self.q_norm is not None
assert self.k_norm is not None
if self.tp_size > 1: if self.tp_size > 1:
q = tensor_model_parallel_all_gather(q.contiguous()) q = tensor_model_parallel_all_gather(q.contiguous())
k = tensor_model_parallel_all_gather(k.contiguous()) k = tensor_model_parallel_all_gather(k.contiguous())
q = self.q_norm(q) q = self.q_norm(q)
k = self.k_norm(k) k = self.k_norm(k)
if self.tp_size > 1: if self.tp_size > 1:
assert self.tp_rank is not None
splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size) splitter = partial(split_tensor_along_last_dim, num_partitions=self.tp_size)
q = splitter(q)[self.tp_rank] q = splitter(q)[self.tp_rank]
k = splitter(k)[self.tp_rank] k = splitter(k)[self.tp_rank]
@@ -1911,10 +1921,10 @@ class Molmo2DummyInputsBuilder(BaseDummyInputsBuilder[Molmo2ProcessingInfo]):
) )
videos = [v.copy() for v in videos] videos = [v.copy() for v in videos]
video_items = [] video_items: list[VideoItem] = []
for video in videos: for video in videos:
video_num_frames = video.shape[0] video_num_frames = video.shape[0]
video_metadata = { video_metadata: dict[str, int | float | str | bool | list[int]] = {
"fps": 2.0, "fps": 2.0,
"duration": video_num_frames / 2.0, "duration": video_num_frames / 2.0,
"total_num_frames": video_num_frames, "total_num_frames": video_num_frames,
@@ -1972,6 +1982,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
image_processor = hf_processor.image_processor image_processor = hf_processor.image_processor
if videos := mm_data.pop("videos", []): if videos := mm_data.pop("videos", []):
assert isinstance(videos, Sequence)
bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id bos_token_id = tokenizer.bos_token_id or tokenizer.eos_token_id
pixel_values_videos_lst = [] pixel_values_videos_lst = []
@@ -1983,7 +1994,9 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
num_video_tokens_lst = [] num_video_tokens_lst = []
for item in videos: for item in videos:
assert isinstance(item, tuple) and len(item) == 2
video_array, metadata = item video_array, metadata = item
assert isinstance(metadata, Mapping)
# NOTE: metadata.frames_indices indicates # NOTE: metadata.frames_indices indicates
# the sampled frames indices of pre-sampled videos, which is # the sampled frames indices of pre-sampled videos, which is
@@ -2240,6 +2253,7 @@ class Molmo2MultiModalProcessor(BaseMultiModalProcessor[Molmo2ProcessingInfo]):
def get_video_replacement_molmo2(item_idx: int): def get_video_replacement_molmo2(item_idx: int):
video, metadata = mm_items["video"][item_idx] video, metadata = mm_items["video"][item_idx]
do_sample_frames = hf_processor_mm_kwargs.get("do_sample_frames") do_sample_frames = hf_processor_mm_kwargs.get("do_sample_frames")
assert do_sample_frames is None or isinstance(do_sample_frames, bool)
timestamps = self.info._get_video_second_idx(metadata, do_sample_frames) timestamps = self.info._get_video_second_idx(metadata, do_sample_frames)
ncols, nrows = self.info.get_base_grid_size(video_processor) ncols, nrows = self.info.get_base_grid_size(video_processor)
@@ -2408,10 +2422,17 @@ class Molmo2ForConditionalGeneration(
image_tokens = kwargs.pop("image_tokens", None) image_tokens = kwargs.pop("image_tokens", None)
num_image_tokens = kwargs.pop("num_image_tokens", None) num_image_tokens = kwargs.pop("num_image_tokens", None)
assert isinstance(token_pooling, torch.Tensor)
assert isinstance(num_pooled_patches, torch.Tensor)
assert isinstance(num_patches, torch.Tensor)
assert isinstance(image_tokens, torch.Tensor)
assert isinstance(num_image_tokens, torch.Tensor)
accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist() accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist()
patch_offset = 0 patch_offset = 0
new_token_pooling = token_pooling.clone() new_token_pooling = token_pooling.clone()
for i, n in enumerate(num_pooled_patches): for i, n_tensor in enumerate(num_pooled_patches):
n = int(n_tensor)
cur_slice = token_pooling[patch_offset : patch_offset + n] cur_slice = token_pooling[patch_offset : patch_offset + n]
index_offset = int(accum_patches[i]) index_offset = int(accum_patches[i])
new_token_pooling[patch_offset : patch_offset + n] = torch.where( new_token_pooling[patch_offset : patch_offset + n] = torch.where(
@@ -2443,10 +2464,17 @@ class Molmo2ForConditionalGeneration(
video_tokens = kwargs.pop("video_tokens", None) video_tokens = kwargs.pop("video_tokens", None)
num_video_tokens = kwargs.pop("num_video_tokens", None) num_video_tokens = kwargs.pop("num_video_tokens", None)
assert isinstance(token_pooling, torch.Tensor)
assert isinstance(num_pooled_patches, torch.Tensor)
assert isinstance(num_patches, torch.Tensor)
assert isinstance(video_tokens, torch.Tensor)
assert isinstance(num_video_tokens, torch.Tensor)
accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist() accum_patches = [0] + num_patches.cumsum(dim=0)[:-1].tolist()
patch_offset = 0 patch_offset = 0
new_token_pooling = token_pooling.clone() new_token_pooling = token_pooling.clone()
for i, n in enumerate(num_pooled_patches): for i, n_tensor in enumerate(num_pooled_patches):
n = int(n_tensor)
cur_slice = token_pooling[patch_offset : patch_offset + n] cur_slice = token_pooling[patch_offset : patch_offset + n]
index_offset = int(accum_patches[i]) index_offset = int(accum_patches[i])
new_token_pooling[patch_offset : patch_offset + n] = torch.where( new_token_pooling[patch_offset : patch_offset + n] = torch.where(
@@ -2464,8 +2492,10 @@ class Molmo2ForConditionalGeneration(
num_video_tokens=num_video_tokens, num_video_tokens=num_video_tokens,
) )
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict: def _parse_and_validate_multimodal_inputs(
modalities = {} self, **kwargs: object
) -> Molmo2MultiModalInputs:
modalities: Molmo2MultiModalInputs = {}
for input_key in kwargs: for input_key in kwargs:
if input_key in ("pixel_values",) and "images" not in modalities: if input_key in ("pixel_values",) and "images" not in modalities:
@@ -2544,10 +2574,12 @@ class Molmo2ForConditionalGeneration(
for modality in modalities: for modality in modalities:
if modality == "images": if modality == "images":
image_input = modalities["images"] image_input = modalities["images"]
assert image_input is not None
image_embeddings = self._process_image_input(image_input) image_embeddings = self._process_image_input(image_input)
multimodal_embeddings += image_embeddings multimodal_embeddings += image_embeddings
if modality == "videos": if modality == "videos":
video_input = modalities["videos"] video_input = modalities["videos"]
assert video_input is not None
video_embeddings = self._process_video_input(video_input) video_embeddings = self._process_video_input(video_input)
multimodal_embeddings += video_embeddings multimodal_embeddings += video_embeddings
+3 -1
View File
@@ -551,9 +551,11 @@ class MoonVitEncoder(nn.Module):
max_seqlen: torch.Tensor | None = None, max_seqlen: torch.Tensor | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
if rope_freqs_cis is None: if rope_freqs_cis is None:
assert grid_hw is not None
rope_freqs_cis = self.rope_2d.get_freqs_cis_by_seqlens(grid_hws=grid_hw) rope_freqs_cis = self.rope_2d.get_freqs_cis_by_seqlens(grid_hws=grid_hw)
if cu_seqlens is None: if cu_seqlens is None:
assert grid_hw is not None
lengths = torch.cat( lengths = torch.cat(
( (
torch.zeros(1, device=hidden_states.device, dtype=grid_hw.dtype), torch.zeros(1, device=hidden_states.device, dtype=grid_hw.dtype),
@@ -578,7 +580,7 @@ class MoonVitEncoder(nn.Module):
def patch_merger( def patch_merger(
x: torch.Tensor, x: torch.Tensor,
grid_hw: torch.Tensor, grid_hw: torch.Tensor,
merge_kernel_size: list[int, int] = (2, 2), merge_kernel_size: tuple[int, int] = (2, 2),
) -> list[torch.Tensor]: ) -> list[torch.Tensor]:
d_model = x.size(-1) d_model = x.size(-1)
+47 -32
View File
@@ -5,6 +5,7 @@
import math import math
from collections.abc import Iterable, Mapping, Sequence from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass, field from dataclasses import dataclass, field
from functools import partial
from typing import Annotated, Any from typing import Annotated, Any
import numpy as np import numpy as np
@@ -14,10 +15,11 @@ import torch.nn.functional as F
from torch import nn from torch import nn
from transformers import BatchFeature, PretrainedConfig, Qwen3Config from transformers import BatchFeature, PretrainedConfig, Qwen3Config
from transformers.models.whisper import WhisperFeatureExtractor from transformers.models.whisper import WhisperFeatureExtractor
from typing_extensions import TypedDict
from vllm.compilation.decorators import support_torch_compile from vllm.compilation.decorators import support_torch_compile
from vllm.config import VllmConfig from vllm.config import VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.distributed import ( from vllm.distributed import (
get_pp_group, get_pp_group,
get_tensor_model_parallel_world_size, get_tensor_model_parallel_world_size,
@@ -89,6 +91,16 @@ DEFAULT_MOSS_AUDIO_MEL_CONFIG = {
"mel_hop_length": 160, "mel_hop_length": 160,
"mel_n_fft": 400, "mel_n_fft": 400,
} }
class MossAudioProcessorKwargs(TypedDict):
audio_token_id: int
audio_start_id: int
audio_end_id: int
enable_time_marker: bool
mel_config: dict[str, int]
MOSS_AUDIO_PLACEHOLDER = ( MOSS_AUDIO_PLACEHOLDER = (
f"{MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN}" f"{MOSS_AUDIO_BOS_TOKEN}{MOSS_AUDIO_TOKEN}{MOSS_AUDIO_EOS_TOKEN}"
) )
@@ -170,7 +182,9 @@ def _extract_moss_audio_mel_config(
for target_key, source_keys in aliases.items(): for target_key, source_keys in aliases.items():
for source_key in source_keys: for source_key in source_keys:
if source_key in mel_config: if source_key in mel_config:
config[target_key] = int(mel_config[source_key]) value = mel_config[source_key]
assert isinstance(value, int)
config[target_key] = value
break break
return config return config
@@ -963,6 +977,9 @@ class MossAudioProcessor:
enable_time_marker: bool = False, enable_time_marker: bool = False,
mel_config: Mapping[str, object] | None = None, mel_config: Mapping[str, object] | None = None,
) -> None: ) -> None:
assert hasattr(tokenizer, "encode")
assert hasattr(tokenizer, "decode")
assert hasattr(tokenizer, "batch_decode")
self.tokenizer = tokenizer self.tokenizer = tokenizer
self.audio_token_id = int(audio_token_id) self.audio_token_id = int(audio_token_id)
self.audio_start_id = int(audio_start_id) self.audio_start_id = int(audio_start_id)
@@ -1229,15 +1246,19 @@ class MossAudioProcessingInfo(BaseProcessingInfo):
@staticmethod @staticmethod
def _get_processor_cache_key(kwargs: Mapping[str, object]) -> tuple[object, ...]: def _get_processor_cache_key(kwargs: Mapping[str, object]) -> tuple[object, ...]:
mel_config = _normalize_moss_audio_mel_config( mel_config_arg = kwargs.get("mel_config")
kwargs.get("mel_config") assert mel_config_arg is None or isinstance(mel_config_arg, Mapping)
if isinstance(kwargs.get("mel_config"), Mapping) mel_config = _normalize_moss_audio_mel_config(mel_config_arg)
else None audio_token_id = kwargs.get("audio_token_id", MOSS_AUDIO_TOKEN_ID)
) audio_start_id = kwargs.get("audio_start_id", MOSS_AUDIO_BOS_TOKEN_ID)
audio_end_id = kwargs.get("audio_end_id", MOSS_AUDIO_EOS_TOKEN_ID)
assert isinstance(audio_token_id, int)
assert isinstance(audio_start_id, int)
assert isinstance(audio_end_id, int)
return ( return (
int(kwargs.get("audio_token_id", MOSS_AUDIO_TOKEN_ID)), audio_token_id,
int(kwargs.get("audio_start_id", MOSS_AUDIO_BOS_TOKEN_ID)), audio_start_id,
int(kwargs.get("audio_end_id", MOSS_AUDIO_EOS_TOKEN_ID)), audio_end_id,
bool(kwargs.get("enable_time_marker", False)), bool(kwargs.get("enable_time_marker", False)),
tuple(sorted(mel_config.items())), tuple(sorted(mel_config.items())),
) )
@@ -1248,21 +1269,19 @@ class MossAudioProcessingInfo(BaseProcessingInfo):
self.ctx.get_merged_mm_kwargs({}), self.ctx.get_merged_mm_kwargs({}),
kwargs, kwargs,
) )
mel_config = _normalize_moss_audio_mel_config( mel_config_arg = merged_kwargs.get("mel_config")
merged_kwargs.get("mel_config") assert mel_config_arg is None or isinstance(mel_config_arg, Mapping)
if isinstance(merged_kwargs.get("mel_config"), Mapping) mel_config = _normalize_moss_audio_mel_config(mel_config_arg)
else None audio_token_id = merged_kwargs.get("audio_token_id", MOSS_AUDIO_TOKEN_ID)
) audio_start_id = merged_kwargs.get("audio_start_id", MOSS_AUDIO_BOS_TOKEN_ID)
processor_kwargs = { audio_end_id = merged_kwargs.get("audio_end_id", MOSS_AUDIO_EOS_TOKEN_ID)
"audio_token_id": int( assert isinstance(audio_token_id, int)
merged_kwargs.get("audio_token_id", MOSS_AUDIO_TOKEN_ID) assert isinstance(audio_start_id, int)
), assert isinstance(audio_end_id, int)
"audio_start_id": int( processor_kwargs: MossAudioProcessorKwargs = {
merged_kwargs.get("audio_start_id", MOSS_AUDIO_BOS_TOKEN_ID) "audio_token_id": audio_token_id,
), "audio_start_id": audio_start_id,
"audio_end_id": int( "audio_end_id": audio_end_id,
merged_kwargs.get("audio_end_id", MOSS_AUDIO_EOS_TOKEN_ID)
),
"enable_time_marker": bool(merged_kwargs.get("enable_time_marker", False)), "enable_time_marker": bool(merged_kwargs.get("enable_time_marker", False)),
"mel_config": mel_config, "mel_config": mel_config,
} }
@@ -1326,6 +1345,7 @@ class MossAudioDummyInputsBuilder(BaseDummyInputsBuilder[MossAudioProcessingInfo
) -> MultiModalDataDict: ) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0) num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return { return {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
length=16000, length=16000,
@@ -1423,7 +1443,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
), ),
) )
prompt_update_specs = [ prompt_update_specs: list[tuple[list[int], list[int]]] = [
( (
[ [
processor.audio_start_id, processor.audio_start_id,
@@ -1452,12 +1472,7 @@ class MossAudioMultiModalProcessor(BaseMultiModalProcessor[MossAudioProcessingIn
PromptReplacement( PromptReplacement(
modality="audio", modality="audio",
target=target, target=target,
replacement=( replacement=partial(get_replacement, suffix_token_ids=suffix_token_ids),
lambda item_idx, suffix_token_ids=suffix_token_ids: get_replacement(
item_idx,
suffix_token_ids,
)
),
) )
for target, suffix_token_ids in prompt_update_specs for target, suffix_token_ids in prompt_update_specs
] ]
@@ -20,7 +20,7 @@ from torch import nn
from transformers import BatchFeature from transformers import BatchFeature
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TextPrompt
from vllm.model_executor.models.interfaces import ( from vllm.model_executor.models.interfaces import (
@@ -299,8 +299,9 @@ class MossTranscribeDiarizeWhisperEncoder(WhisperEncoder):
def forward( def forward(
self, self,
input_features: torch.Tensor, input_features: torch.Tensor,
audio_feature_lengths: torch.Tensor, audio_feature_lengths: torch.Tensor | None = None,
) -> torch.Tensor: ) -> torch.Tensor:
assert audio_feature_lengths is not None
if input_features.numel() == 0: if input_features.numel() == 0:
return input_features.new_empty((1, 0, self.conv1.out_channels)) return input_features.new_empty((1, 0, self.conv1.out_channels))
device = self.conv1.weight.device device = self.conv1.weight.device
@@ -431,11 +432,13 @@ class MossTranscribeDiarizeDummyInputsBuilder(
return {} return {}
feature_extractor = self.info.get_feature_extractor() feature_extractor = self.info.get_feature_extractor()
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return { return {
"audio": self._get_dummy_audios( "audio": self._get_dummy_audios(
length=_get_max_audio_samples(feature_extractor), length=_get_max_audio_samples(feature_extractor),
num_audios=num_audios, num_audios=num_audios,
overrides=mm_options.get("audio"), overrides=audio_overrides,
) )
} }
+21 -7
View File
@@ -45,7 +45,11 @@ from transformers import BatchFeature
from vllm.compilation.decorators import support_torch_compile from vllm.compilation.decorators import support_torch_compile
from vllm.config import CacheConfig, VllmConfig from vllm.config import CacheConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions from vllm.config.multimodal import (
BaseDummyOptions,
ImageDummyOptions,
VideoDummyOptions,
)
from vllm.distributed import ( from vllm.distributed import (
divide, divide,
get_pp_group, get_pp_group,
@@ -241,12 +245,16 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
* int(video_processor.downsample_factor) * int(video_processor.downsample_factor)
* video_grid * video_grid
) )
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert image_overrides is None or isinstance(image_overrides, ImageDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return { return {
"image": self._get_dummy_images( "image": self._get_dummy_images(
width=image_width, width=image_width,
height=image_height, height=image_height,
num_images=mm_counts.get("image", 0), num_images=mm_counts.get("image", 0),
overrides=mm_options.get("image"), overrides=image_overrides,
), ),
"video": self._get_dummy_videos( "video": self._get_dummy_videos(
width=video_size, width=video_size,
@@ -255,7 +263,7 @@ class MuseGlimmerDummyInputsBuilder(BaseDummyInputsBuilder[MuseGlimmerProcessing
seq_len, mm_counts seq_len, mm_counts
), ),
num_videos=mm_counts.get("video", 0), num_videos=mm_counts.get("video", 0),
overrides=mm_options.get("video"), overrides=video_overrides,
), ),
} }
@@ -370,7 +378,7 @@ class MuseGlimmerMultiModalProcessor(
hf_inputs: BatchFeature, hf_inputs: BatchFeature,
hf_processor_mm_kwargs: Mapping[str, object], hf_processor_mm_kwargs: Mapping[str, object],
) -> Mapping[str, MultiModalFieldConfig]: ) -> Mapping[str, MultiModalFieldConfig]:
fields = {} fields: dict[str, MultiModalFieldConfig] = {}
if "image_pixel_values" in hf_inputs: if "image_pixel_values" in hf_inputs:
fields.update( fields.update(
image_pixel_values=MultiModalFieldConfig.batched("image"), image_pixel_values=MultiModalFieldConfig.batched("image"),
@@ -410,7 +418,9 @@ class MuseGlimmerMultiModalProcessor(
def image_replacement(item_idx: int) -> PromptUpdateDetails: def image_replacement(item_idx: int) -> PromptUpdateDetails:
out_item = out_mm_kwargs["image"][item_idx] out_item = out_mm_kwargs["image"][item_idx]
num_tokens = int(out_item["image_feature_sizes"].data) feature_size = out_item["image_feature_sizes"].data
assert isinstance(feature_size, torch.Tensor)
num_tokens = int(feature_size.item())
replacement = ( replacement = (
[image_start_id] + [config.image_token_id] * num_tokens + [image_end_id] [image_start_id] + [config.image_token_id] * num_tokens + [image_end_id]
) )
@@ -427,7 +437,9 @@ class MuseGlimmerMultiModalProcessor(
"MuseGlimmer video must contain at least one frame group" "MuseGlimmer video must contain at least one frame group"
) )
num_tokens = int(out_item["video_feature_sizes"].data) feature_size = out_item["video_feature_sizes"].data
assert isinstance(feature_size, torch.Tensor)
num_tokens = int(feature_size.item())
tokens_per_group, remainder = divmod(num_tokens, num_groups) tokens_per_group, remainder = divmod(num_tokens, num_groups)
if remainder: if remainder:
raise ValueError( raise ValueError(
@@ -1477,6 +1489,8 @@ class MuseGlimmerForCausalLM(
prefix=maybe_prefix(prefix, "model"), prefix=maybe_prefix(prefix, "model"),
) )
self.vision_encoder: MuseGlimmerVisionEncoder | None
self.vision_adapter: MuseGlimmerVisionAdapter | None
if self.has_vision: if self.has_vision:
with self._mark_tower_model(vllm_config, {"image", "video"}): with self._mark_tower_model(vllm_config, {"image", "video"}):
self.vision_encoder = MuseGlimmerVisionEncoder( self.vision_encoder = MuseGlimmerVisionEncoder(
@@ -1616,7 +1630,7 @@ class MuseGlimmerForCausalLM(
image_input = self._parse_and_validate_image_input(**kwargs) image_input = self._parse_and_validate_image_input(**kwargs)
video_input = self._parse_and_validate_video_input(**kwargs) video_input = self._parse_and_validate_video_input(**kwargs)
embeddings: MultiModalEmbeddings = [] embeddings: list[torch.Tensor] = []
for key in kwargs: for key in kwargs:
if key == "image_pixel_values" and image_input is not None: if key == "image_pixel_values" and image_input is not None:
embeddings.extend(self._process_image_input(image_input)) embeddings.extend(self._process_image_input(image_input))