[Mypy Fix] Mypy fix for "vllm/model_executor/models/[gG]" (#53616)

Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
Wentao Ye
2026-08-25 09:47:16 -07:00
committed by GitHub
parent 217a0b5cef
commit 70021a0081
26 changed files with 350 additions and 123 deletions
-1
View File
@@ -112,7 +112,6 @@ SEPARATE_GROUPS = [
]
EXCLUDE = [
r"vllm/model_executor/models/[gG]",
r"vllm/model_executor/models/[hH]",
r"vllm/model_executor/models/[iI]",
r"vllm/model_executor/models/[jJ]",
+2 -2
View File
@@ -457,7 +457,7 @@ class DeepseekV2Attention(nn.Module):
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: int,
q_lora_rank: int | None,
kv_lora_rank: int,
max_position_embeddings: int = 8192,
cache_config: CacheConfig | None = None,
@@ -495,7 +495,7 @@ class DeepseekV2Attention(nn.Module):
)
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
self.q_b_proj = ColumnParallelLinear(
q_lora_rank,
self.q_lora_rank,
self.num_heads * self.qk_head_dim,
bias=False,
quant_config=quant_config,
+1
View File
@@ -311,6 +311,7 @@ class GemmaModel(nn.Module):
hidden_states = self.embed_input_ids(input_ids)
residual = None
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
residual = intermediate_tensors["residual"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
+2
View File
@@ -852,7 +852,9 @@ class Gemma3ForConditionalGeneration(
def encoder_eager_forward(
self,
mm_kwargs: dict[str, Any],
path: str = "default",
) -> torch.Tensor:
image_input = self._parse_and_validate_image_input(**mm_kwargs)
assert image_input is not None
results = self._process_image_input(image_input)
return torch.cat(results, dim=0)
+1 -1
View File
@@ -77,7 +77,7 @@ class Gemma3nAltUp(nn.Module):
altup_num_inputs: int,
altup_coef_clip: float,
altup_active_idx: int,
quant_config: QuantizationConfig,
quant_config: QuantizationConfig | None,
prefix: str,
):
super().__init__()
+31 -10
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import inspect
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Iterable, Mapping, Sequence, Sized
from typing import Annotated, Any, Literal
import torch
@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (
from transformers.models.siglip import SiglipImageProcessorFast
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
from vllm.logger import init_logger
@@ -115,8 +115,10 @@ def batch_audio_features(
The `(bn, s_max, f)` features and their `(bn, s_max)` mask.
"""
if isinstance(input_features, torch.Tensor):
assert isinstance(input_features_mask, torch.Tensor)
return input_features.squeeze(1), input_features_mask.squeeze(1)
assert isinstance(input_features_mask, list)
max_len = max(features.shape[0] for features in input_features)
batched_features = input_features[0].new_zeros(
(len(input_features), max_len, input_features[0].shape[-1])
@@ -248,6 +250,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
image_overrides = mm_options.get("image")
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"image": self._get_dummy_images(
@@ -293,10 +296,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
# output (and thus its cache entry) is independent of the other
# items in the batch. `batch_audio_features` re-pads the
# per-item features so audio_tower can still run batched.
num_frames = [
self._get_num_audio_frames_alone(len(audio))
for audio in mm_data["audio"]
]
audios = mm_data["audio"]
if not isinstance(audios, Sequence):
raise TypeError("audio data must be a sequence")
num_frames = []
for audio in audios:
if not isinstance(audio, Sized):
raise TypeError("each audio item must have a length")
num_frames.append(self._get_num_audio_frames_alone(len(audio)))
processed_data["input_features_padded"] = [
features[:n]
for features, n in zip(processed_data.pop("input_features"), num_frames)
@@ -685,7 +692,9 @@ class Gemma3nForConditionalGeneration(
)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
mm_input_by_modality: dict[
str, Gemma3nImageInputs | Gemma3nAudioInputs | None
] = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -852,7 +861,13 @@ class Gemma3nForConditionalGeneration(
# select a chunk of pre-allocated PLEs. During normal execution,
# `embed_input_ids` is called before forward, hence this slice
# will contain PLEs computed from the actual input_ids.
per_layer_inputs = self.per_layer_embeddings[: inputs_embeds.shape[0]]
if inputs_embeds is not None:
num_tokens = inputs_embeds.shape[0]
elif intermediate_tensors is not None:
num_tokens = intermediate_tensors["hidden_states"].shape[0]
else:
raise ValueError("inputs_embeds is required on the first PP rank")
per_layer_inputs = self.per_layer_embeddings[:num_tokens]
hidden_states = self.language_model.model(
input_ids,
@@ -913,9 +928,15 @@ class Gemma3nForConditionalGeneration(
prompt += " this audio"
# We assume the language is a valid ISO 639-1 code.
full_lang_name = cls.supported_languages.get(language, "")
full_lang_name = (
cls.supported_languages.get(language, "") if language is not None else ""
)
# Translation only for now
full_lang_name_to = cls.supported_languages.get(to_language, "")
full_lang_name_to = (
cls.supported_languages.get(to_language, "")
if to_language is not None
else ""
)
if task_type == "transcribe" and full_lang_name:
prompt += f" into {full_lang_name}"
+31 -7
View File
@@ -352,15 +352,19 @@ class Gemma4MoE(nn.Module):
)
# MoERunner experts with custom Gemma4 routing
intermediate_size = getattr(
config,
"moe_intermediate_size",
getattr(config, "expert_intermediate_size", None),
)
if intermediate_size is None:
raise ValueError("Gemma4 MoE requires an expert intermediate size")
self.experts = FusedMoEFactory(
num_experts=config.num_experts,
top_k=config.top_k_experts,
hidden_size=config.hidden_size,
intermediate_size=getattr(
config,
"moe_intermediate_size",
getattr(config, "expert_intermediate_size", None),
),
intermediate_size=intermediate_size,
renormalize=True,
quant_config=quant_config,
prefix=f"{prefix}.experts",
@@ -624,6 +628,11 @@ class Gemma4DecoderLayer(nn.Module):
self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr(
config, "use_second_mlp_block", False
)
self.router: Gemma4Router | None
self.moe: Gemma4MoE | None
self.post_feedforward_layernorm_1: RMSNorm | None
self.post_feedforward_layernorm_2: RMSNorm | None
self.pre_feedforward_layernorm_2: RMSNorm | None
if self.enable_moe_block:
self.router = Gemma4Router(
config,
@@ -652,6 +661,9 @@ class Gemma4DecoderLayer(nn.Module):
self.pre_feedforward_layernorm_2 = None
# Per-Layer Embedding (PLE) components — present in each decoder layer
self.per_layer_input_gate: ReplicatedLinear | None
self.per_layer_projection: ReplicatedLinear | None
self.post_per_layer_input_norm: RMSNorm | None
if (
self.hidden_size_per_layer_input is not None
and self.hidden_size_per_layer_input > 0
@@ -716,6 +728,11 @@ class Gemma4DecoderLayer(nn.Module):
hidden_states = self.mlp(hidden_states)
if self.enable_moe_block:
assert self.post_feedforward_layernorm_1 is not None
assert self.pre_feedforward_layernorm_2 is not None
assert self.router is not None
assert self.moe is not None
assert self.post_feedforward_layernorm_2 is not None
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states)
hidden_states_2 = self.pre_feedforward_layernorm_2(residual)
@@ -731,6 +748,8 @@ class Gemma4DecoderLayer(nn.Module):
# Apply PLE (Per-Layer Embedding) if configured
if per_layer_input is not None and self.per_layer_input_gate is not None:
assert self.per_layer_projection is not None
assert self.post_per_layer_input_norm is not None
gate = self.per_layer_input_gate(hidden_states)
gate = torch.nn.functional.gelu(gate, approximate="tanh")
gated_per_layer = gate * per_layer_input
@@ -865,6 +884,7 @@ class Gemma4SelfDecoderLayers(nn.Module):
"""
if self.per_layer_model_projection is None:
return None
assert self.per_layer_projection_norm is not None
per_layer_projection = self.per_layer_model_projection(inputs_embeds)
per_layer_projection = per_layer_projection * self.per_layer_projection_scale
per_layer_projection = per_layer_projection.reshape(
@@ -971,6 +991,9 @@ class Gemma4Model(nn.Module, EagleModelMixin):
)
# Per-Layer Embedding (PLE) components
self.embed_tokens_per_layer: VocabParallelEmbedding | None
self.per_layer_model_projection: ColumnParallelLinear | None
self.per_layer_projection_norm: RMSNorm | None
if (
self.hidden_size_per_layer_input is not None
and self.hidden_size_per_layer_input > 0
@@ -1472,9 +1495,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
else:
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
if is_pp_missing_parameter(name, self):
continue
# Skip if name doesn't exist in params_dict (e.g., individual
+5 -1
View File
@@ -89,7 +89,11 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
) -> tuple[torch.Tensor, torch.Tensor]:
k, _ = self.k_proj(hidden_states)
k_normed = self.k_norm(k.unflatten(-1, (self.num_kv_heads, self.head_dim)))
v_src = k if self.use_k_eq_v else self.v_proj(hidden_states)[0]
if self.use_k_eq_v:
v_src = k
else:
assert self.v_proj is not None
v_src = self.v_proj(hidden_states)[0]
v_normed = self.v_norm(v_src.unflatten(-1, (self.num_kv_heads, self.head_dim)))
return k_normed.flatten(-2, -1), v_normed.flatten(-2, -1)
+58 -14
View File
@@ -35,7 +35,11 @@ from transformers.models.gemma4.configuration_gemma4 import (
from vllm.config import VllmConfig
from vllm.config.model import get_served_model_name
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
from vllm.config.multimodal import (
AudioDummyOptions,
BaseDummyOptions,
VideoDummyOptions,
)
from vllm.inputs import MultiModalDataDict
from vllm.logger import init_logger
from vllm.model_executor.layers.layernorm import RMSNorm
@@ -189,6 +193,7 @@ class Gemma4VideoInputs(TensorSchema):
torch.Tensor,
TensorShape("bn", "np", 2),
]
video_frame_counts: Annotated[torch.Tensor, TensorShape("nv")]
# ---------------------------------------------------------------------------
@@ -490,8 +495,10 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
image_overrides = mm_options.get("image") if mm_options else None
audio_overrides = mm_options.get("audio") if mm_options else None
video_overrides = mm_options.get("video") if mm_options else None
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
data: MultiModalDataDict = {
data: dict[str, Any] = {
"image": self._get_dummy_images(
width=img_width,
height=img_height,
@@ -592,6 +599,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
# same vision tower, matching transformers processing_gemma4.py.
video_outputs: dict[str, Any] = {}
if videos := mm_data.pop("videos", []):
assert isinstance(videos, list)
processor = self.info.get_hf_processor()
all_video_pixel_values: list[torch.Tensor] = []
@@ -858,8 +866,29 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
def get_replacement_video(item_idx: int):
out_item = out_mm_kwargs["video"][item_idx]
timestamps = out_item["video_timestamps"].data
num_soft = out_item["video_num_soft_tokens"].data
timestamps_data = out_item["video_timestamps"].data
if isinstance(timestamps_data, torch.Tensor):
timestamps_data = timestamps_data.tolist()
if not isinstance(timestamps_data, list):
raise TypeError("video timestamps must be a list")
timestamps: list[float] = []
for timestamp in timestamps_data:
if not isinstance(timestamp, (int, float)):
raise TypeError("each video timestamp must be numeric")
timestamps.append(float(timestamp))
num_soft_data = out_item["video_num_soft_tokens"].data
if isinstance(num_soft_data, torch.Tensor):
num_soft_data = num_soft_data.tolist()
if not isinstance(num_soft_data, list):
raise TypeError("video soft-token counts must be a list")
num_soft: list[int] = []
for count in num_soft_data:
if not isinstance(count, int):
raise TypeError(
"each video soft-token count must be an integer"
)
num_soft.append(count)
return self.info.get_video_repl(
timestamps=timestamps,
num_soft_tokens_per_frame=num_soft,
@@ -1038,6 +1067,7 @@ class Gemma4ForConditionalGeneration(
# (Marlin, FP8, …) require dimensions divisible by 64, which
# the vision tower (intermediate_size=4304) does not satisfy.
# TODO(mgoin): remove this by fixing kernel padding.
tower_quant: QuantizationConfig | None
if quant_config and quant_config.get_name() in [
"bitsandbytes",
"torchao",
@@ -1068,6 +1098,7 @@ class Gemma4ForConditionalGeneration(
)
# ---- Audio tower (variants with audio_config) ----
self.embed_audio: Gemma4MultimodalEmbedder | None
if config.audio_config is not None:
with self._mark_tower_model(vllm_config, "audio"):
self.audio_tower = AutoModel.from_config(config=config.audio_config)
@@ -1177,22 +1208,24 @@ class Gemma4ForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> dict[str, torch.Tensor] | None:
) -> Gemma4VideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
pixel_position_ids_videos = kwargs.pop("pixel_position_ids_videos", None)
video_frame_counts = kwargs.pop("video_frame_counts", None)
if pixel_values_videos is None:
return None
return {
"pixel_values_videos": pixel_values_videos,
"pixel_position_ids_videos": pixel_position_ids_videos,
"video_frame_counts": video_frame_counts,
}
return Gemma4VideoInputs(
pixel_values_videos=pixel_values_videos,
pixel_position_ids_videos=pixel_position_ids_videos,
video_frame_counts=video_frame_counts,
)
def _parse_and_validate_multimodal_inputs(
self, **kwargs: object
) -> dict[str, Gemma4ImageInputs | Gemma4AudioInputs | Gemma4VideoInputs | None]:
mm_input_by_modality = {}
mm_input_by_modality: dict[
str, Gemma4ImageInputs | Gemma4AudioInputs | Gemma4VideoInputs | None
] = {}
for input_key in list(kwargs):
if (
input_key in ("pixel_values", "image_embeds")
@@ -1419,7 +1452,7 @@ class Gemma4ForConditionalGeneration(
def _process_video_input(
self,
video_input: dict[str, torch.Tensor],
video_input: Gemma4VideoInputs,
) -> list[torch.Tensor]:
"""Batch-encode video frames through the vision tower.
@@ -1543,6 +1576,8 @@ class Gemma4ForConditionalGeneration(
audio_input["input_features_mask"],
)
assert self.audio_tower is not None
assert self.embed_audio is not None
# Run audio tower — mask convention: True=valid, False=padding.
audio_outputs = self.audio_tower(input_features, input_features_mask)
if isinstance(audio_outputs, tuple):
@@ -1576,14 +1611,17 @@ class Gemma4ForConditionalGeneration(
if multimodal_input is None:
continue
if modality == "image":
assert isinstance(multimodal_input, Gemma4ImageInputs)
multimodal_embeddings.extend(
self._process_image_input(multimodal_input)
)
elif modality == "video":
assert isinstance(multimodal_input, Gemma4VideoInputs)
multimodal_embeddings.extend(
self._process_video_input(multimodal_input)
)
elif modality == "audio":
assert isinstance(multimodal_input, Gemma4AudioInputs)
multimodal_embeddings.extend(
self._process_audio_input(multimodal_input)
)
@@ -2204,13 +2242,15 @@ class Gemma4ForConditionalGeneration(
mm_kwargs: MultiModalKwargsItem | None,
num_mm_embeds: int,
) -> tuple[int, int | None]:
tower_tokens: int | None = None
connector_tokens: int | None = None
if modality in ("image", "video"):
vision_config = self.config.vision_config
pooling_k2 = vision_config.pooling_kernel_size**2
if modality == "image":
pixel_values_key = "pixel_values"
max_soft_tokens = vision_config.default_output_length
max_soft_tokens = int(vision_config.default_output_length)
mm_processor_kwargs = getattr(
getattr(self, "multimodal_config", None),
"mm_processor_kwargs",
@@ -2241,7 +2281,7 @@ class Gemma4ForConditionalGeneration(
* pooling_k2
)
if modality == "audio":
elif modality == "audio":
tower_tokens = num_mm_embeds
connector_tokens = num_mm_embeds
@@ -2255,6 +2295,10 @@ class Gemma4ForConditionalGeneration(
tower_tokens = audio_tokens
connector_tokens = audio_tokens
else:
raise ValueError(f"Unsupported modality: {modality}")
assert tower_tokens is not None
return tower_tokens, connector_tokens
@classmethod
+10 -3
View File
@@ -345,7 +345,9 @@ class Gemma4MultiTokenPredictor(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.speculative_config.draft_model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
text_config = _get_text_config(config)
quant_config = get_draft_quant_config(vllm_config)
self.config = text_config
@@ -467,7 +469,9 @@ class Gemma4MTP(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.speculative_config.draft_model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
text_config = _get_text_config(config)
self.quant_config = get_draft_quant_config(vllm_config)
self.config = config
@@ -496,6 +500,7 @@ class Gemma4MTP(nn.Module):
soft_cap=getattr(text_config, "final_logit_softcapping", None),
)
self.masked_embedding: Gemma4MTPMaskedEmbedder | None
if getattr(config, "use_ordered_embeddings", False):
num_centroids = getattr(config, "num_centroids", 2048)
top_k = getattr(config, "centroid_intermediate_top_k", 32)
@@ -516,7 +521,7 @@ class Gemma4MTP(nn.Module):
else:
self.masked_embedding = None
draft_cfg = vllm_config.speculative_config.draft_model_config
draft_cfg = speculative_config.draft_model_config
gen_cfg = draft_cfg.try_get_generation_config()
self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
@@ -545,6 +550,7 @@ class Gemma4MTP(nn.Module):
def _get_full_lm_head_weight(self) -> torch.Tensor:
if self._stable_full_lm_head_weight is not None:
return self._stable_full_lm_head_weight
assert self.masked_embedding is not None
lm_head_weight = self.lm_head.weight
tp_size = get_tensor_model_parallel_world_size()
if tp_size > 1:
@@ -579,6 +585,7 @@ class Gemma4MTP(nn.Module):
hidden_states: torch.Tensor,
) -> torch.Tensor:
"""Sparse argmax via centroids masking. Returns token IDs directly."""
assert self.masked_embedding is not None
return self.masked_embedding.get_top_tokens(
hidden_states,
self._get_full_lm_head_weight(),
+12 -16
View File
@@ -44,6 +44,7 @@ from vllm.model_executor.models.gemma4_mm import (
Gemma4MultimodalEmbedder,
Gemma4MultiModalProcessor,
Gemma4ProcessingInfo,
Gemma4VideoInputs,
_get_max_soft_tokens,
)
from vllm.model_executor.models.module_mapping import MultiModelKeys
@@ -265,22 +266,16 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
self.audio_tower = None
# ---- Encoder-free vision embedder ----
self.vision_embedder = (
Gemma4UnifiedVisionEmbedder(
config.vision_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "vision_embedder"),
)
if config.vision_config is not None
else None
if config.vision_config is None:
raise ValueError("Gemma4 Unified requires a vision configuration")
self.vision_embedder = Gemma4UnifiedVisionEmbedder(
config.vision_config,
quant_config=quant_config,
prefix=maybe_prefix(prefix, "vision_embedder"),
)
self.embed_vision = (
Gemma4MultimodalEmbedder(
config.vision_config,
config.text_config,
)
if config.vision_config is not None
else None
self.embed_vision = Gemma4MultimodalEmbedder(
config.vision_config,
config.text_config,
)
# ---- Encoder-free audio embedder ----
@@ -382,7 +377,7 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
def _process_video_input(
self,
video_input: dict[str, torch.Tensor],
video_input: Gemma4VideoInputs,
) -> list[torch.Tensor]:
"""Project video frames to LM space, one frame at a time.
@@ -429,6 +424,7 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
audio_input["input_features_mask"],
)
assert self.embed_audio is not None
target_dtype = self.embed_audio.embedding_projection.weight.dtype
audio_features = self.embed_audio(input_features.to(target_dtype))
per_audio: list[torch.Tensor] = []
+43 -15
View File
@@ -1393,9 +1393,15 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
return list(timestamps_list)
def _get_video_frame_embed_token_id(self, hf_processor: object) -> int:
if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA:
return hf_processor.image_token_id
return hf_processor.video_token_id
attr = (
"image_token_id"
if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA
else "video_token_id"
)
token_id = getattr(hf_processor, attr, None)
if not isinstance(token_id, int):
raise ValueError(f"Processor has no valid {attr}")
return token_id
def _construct_video_placeholder(
self,
@@ -1479,6 +1485,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
image_overrides = mm_options.get("image")
video_overrides = mm_options.get("video")
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
return {
"image": self._get_dummy_images(
@@ -1519,7 +1526,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
)
videos = [v.copy() for v in videos]
video_items = []
video_items: list[VideoItem] = []
for video in videos:
video_num_frames = video.shape[0]
video_metadata = {
@@ -1611,16 +1618,14 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
processed_data.update(passthrough_data)
return processed_data
if (
"videos" in mm_data
and isinstance(mm_data["videos"], list)
and len(mm_data["videos"]) > 0
):
videos = mm_data.get("videos")
if isinstance(videos, list) and videos:
video_grid_thw_lst = []
pixel_values_videos_lst = []
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
for item in mm_data.pop("videos", []):
mm_data.pop("videos")
for item in videos:
video_array, metadata = item
# don't update mm_kwargs inplace
@@ -1794,6 +1799,7 @@ class Glm4vForConditionalGeneration(
self.config = config
self.model_config = vllm_config.model_config
self.multimodal_config = multimodal_config
assert multimodal_config is not None
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
self.is_multimodal_pruning_enabled = (
multimodal_config.is_multimodal_pruning_enabled()
@@ -1850,6 +1856,7 @@ class Glm4vForConditionalGeneration(
image_embeds=image_embeds,
image_grid_thw=image_grid_thw,
)
raise AssertionError("image input must contain pixels or embeddings")
def _parse_and_validate_video_input(
self, **kwargs: object
@@ -1874,6 +1881,7 @@ class Glm4vForConditionalGeneration(
video_embeds=video_embeds,
video_grid_thw=video_grid_thw,
)
raise AssertionError("video input must contain pixels or embeddings")
def _process_image_input(
self, image_input: Glm4vImageInputs
@@ -1996,12 +2004,12 @@ class Glm4vForConditionalGeneration(
def _get_grid_thw_by_modality(
self,
mm_kwargs: dict[str, Any],
) -> list[tuple[int, int, int]]:
) -> list[list[int]]:
grid_thw_key = f"{self.get_input_modality(mm_kwargs)}_grid_thw"
grid_thw = mm_kwargs[grid_thw_key]
if not isinstance(grid_thw, list):
grid_thw = grid_thw.tolist()
return grid_thw
return [list(row) for row in grid_thw]
def get_encoder_cudagraph_item_specs(
self,
@@ -2188,7 +2196,7 @@ class Glm4vForConditionalGeneration(
return self.visual(pixel_values, grid_thw)
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
mm_input_by_modality = {}
mm_input_by_modality: dict[str, Glm4vImageInputs | Glm4vVideoInputs | None] = {}
# Preserve the order of modalities if there are multiple of them
# from the order of kwargs.
@@ -2223,9 +2231,17 @@ class Glm4vForConditionalGeneration(
for modality in mm_input_by_modality:
multimodal_input = mm_input_by_modality[modality]
if modality == "image":
assert isinstance(
multimodal_input,
(Glm4vImagePixelInputs, Glm4vImageEmbeddingInputs),
)
image_embeddings = self._process_image_input(multimodal_input)
multimodal_embeddings += tuple(image_embeddings)
if modality == "video":
assert isinstance(
multimodal_input,
(Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs),
)
video_embeddings = self._process_video_input(multimodal_input)
multimodal_embeddings += tuple(video_embeddings)
return multimodal_embeddings
@@ -2238,14 +2254,26 @@ class Glm4vForConditionalGeneration(
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
embed_ranges = mm_feature.mm_position.extract_embeds_range()
if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
feature_data = mm_feature.data
assert feature_data is not None
grid_item = feature_data.get("image_grid_thw")
assert grid_item is not None
grid_data = grid_item.data
assert isinstance(grid_data, torch.Tensor)
t, h, w = grid_data.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
assert len(embed_ranges) == 1
offset, end = embed_ranges[0]
assert end - offset + 1 == h * w // spatial_merge_size**2
yield offset, t, h // spatial_merge_size, w // spatial_merge_size
elif mm_feature.modality == "video":
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
feature_data = mm_feature.data
assert feature_data is not None
grid_item = feature_data.get("video_grid_thw")
assert grid_item is not None
grid_data = grid_item.data
assert isinstance(grid_data, torch.Tensor)
t, h, w = grid_data.tolist()
llm_grid_h = h // spatial_merge_size
llm_grid_w = w // spatial_merge_size
num_tokens_per_frame = llm_grid_h * llm_grid_w
+2
View File
@@ -502,6 +502,8 @@ class Glm4MoeModel(nn.Module):
class Glm4MixtureOfExperts(MixtureOfExperts):
moe_mlp_layers: list[Glm4MoE]
def extract_moe_parameters(self, example_moe: Glm4MoE | None) -> None:
if example_moe is None:
raise RuntimeError("No Glm4MoE layer found in model.layers.")
+10 -10
View File
@@ -74,7 +74,6 @@ from .utils import (
PPMissingLayer,
get_spec_layer_idx_from_weight_name,
is_pp_missing_parameter,
make_empty_intermediate_tensors_factory,
make_layers,
maybe_prefix,
)
@@ -132,6 +131,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
v_head_dim = getattr(config, "v_head_dim", 0)
kv_lora_rank = getattr(config, "kv_lora_rank", 0)
attn_cls: Callable[..., Glm4MoeLiteMLAAttention | Glm4MoeLiteAttention]
if model_config.use_mla:
attn_cls = Glm4MoeLiteMLAAttention
else:
@@ -145,7 +145,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
qk_nope_head_dim=qk_nope_head_dim,
qk_rope_head_dim=qk_rope_head_dim,
v_head_dim=v_head_dim,
q_lora_rank=config.q_lora_rank if hasattr(config, "q_lora_rank") else None,
q_lora_rank=getattr(config, "q_lora_rank", None),
kv_lora_rank=kv_lora_rank,
max_position_embeddings=max_position_embeddings,
cache_config=cache_config,
@@ -266,9 +266,6 @@ class Glm4MoeLiteModel(nn.Module):
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
else:
self.norm = PPMissingLayer()
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
["hidden_states", "residual"], config.hidden_size
)
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.embed_tokens(input_ids)
@@ -454,7 +451,9 @@ class Glm4MoeLiteModel(nn.Module):
# param and delegate to its expert-aware weight_loader
# with expert_id.
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
param_name, weight_name, mapping_expert_id, mapping_shard_id = (
mapping
)
if weight_name not in chunk_name:
continue
@@ -480,8 +479,8 @@ class Glm4MoeLiteModel(nn.Module):
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
shard_id=mapping_shard_id,
expert_id=mapping_expert_id,
return_success=True,
)
if success:
@@ -502,9 +501,10 @@ class Glm4MoeLiteModel(nn.Module):
continue
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
if is_pp_missing_parameter(name, self):
continue
@@ -85,7 +85,9 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
super().__init__()
config = vllm_config.speculative_config.draft_model_config.hf_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config
self.config = config
quant_config = vllm_config.quant_config
@@ -235,7 +237,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor | None,
positions: torch.Tensor,
@@ -370,7 +372,9 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
# with expert_id.
is_expert_weight = False
for mapping in expert_params_mapping:
param_name, weight_name, expert_id, shard_id = mapping
param_name, weight_name, mapping_expert_id, mapping_shard_id = (
mapping
)
if weight_name not in chunk_name:
continue
@@ -393,8 +397,8 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
param,
weight_to_load,
name_mapped,
shard_id=shard_id,
expert_id=expert_id,
shard_id=mapping_shard_id,
expert_id=mapping_expert_id,
return_success=True,
)
if success:
@@ -414,9 +418,10 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
if name.endswith(".bias") and name not in params_dict:
continue
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
# According to DeepSeek-V3 Technical Report, MTP modules
# shares embedding layer. We only load the first weights.
@@ -93,6 +93,7 @@ class Glm4MoeMultiTokenPredictorLayer(nn.Module):
self.shared_head = SharedHead(
config=config, prefix=prefix, quant_config=quant_config
)
assert parallel_config is not None
self.enable_eplb = parallel_config.enable_eplb
self.mtp_block = Glm4MoeDecoderLayer(
config=config,
+19 -2
View File
@@ -582,7 +582,13 @@ class GLM4VForCausalLM(
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
offset = mm_feature.mm_position.offset
if mm_feature.modality == "image":
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
feature_data = mm_feature.data
assert feature_data is not None
grid_item = feature_data.get("image_grid_thw")
assert grid_item is not None
grid_data = grid_item.data
assert isinstance(grid_data, torch.Tensor)
t, h, w = grid_data.tolist()
assert t == 1, f"Image must have 1 frame, got {t}"
yield offset, t, h // spatial_merge_size, w // spatial_merge_size
else:
@@ -625,7 +631,18 @@ class GLM4VForCausalLM(
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
return torch.from_numpy(llm_positions), mrope_position_delta
embed_input_ids = SupportsMultiModal.embed_input_ids
def embed_input_ids(
self,
input_ids: torch.Tensor,
multimodal_embeddings: MultiModalEmbeddings | None = None,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
return SupportsMultiModal.embed_input_ids(
self,
input_ids,
multimodal_embeddings,
is_multimodal=is_multimodal,
)
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
image_input = self._parse_and_validate_image_input(**kwargs)
+2
View File
@@ -326,7 +326,9 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
self,
x: torch.Tensor,
grid_thw: torch.Tensor | list[list[int]],
encoder_metadata: dict[str, torch.Tensor] | None = None,
) -> torch.Tensor:
del encoder_metadata
if isinstance(grid_thw, list):
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
+7 -4
View File
@@ -58,7 +58,9 @@ class GlmOcrMultiTokenPredictorLayer(nn.Module):
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
nn.Module.__init__(self)
config = vllm_config.speculative_config.draft_model_config.hf_config.text_config
speculative_config = vllm_config.speculative_config
assert speculative_config is not None
config = speculative_config.draft_model_config.hf_config.text_config
self.config = config
quant_config = vllm_config.quant_config
@@ -144,7 +146,7 @@ class GlmOcrMTP(nn.Module, SupportsPP):
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.model.embed_input_ids(input_ids)
def forward(
def forward( # type: ignore[override]
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
@@ -195,9 +197,10 @@ class GlmOcrMTP(nn.Module, SupportsPP):
if "scale" in name or "zero_point" in name:
# Remapping the name of FP8 kv-scale or zero point.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
for param_name, weight_name, shard_id in stacked_params_mapping:
if weight_name not in name:
+15 -4
View File
@@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor
from transformers.models.whisper import WhisperFeatureExtractor
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
@@ -710,6 +710,7 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
sampling_rate = feature_extractor.sampling_rate
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
max_audio_len = getattr(
self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
@@ -793,8 +794,13 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
)
processed_data["feature_attention_mask"] = mask
audio = mm_data.get("audio", [])
audio_list = [audio] if audio and not isinstance(audio, list) else audio
audio = mm_data.get("audio")
if audio is None:
audio_list: list[Any] = []
elif isinstance(audio, list):
audio_list = audio
else:
audio_list = [audio]
if audio_list:
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
@@ -878,6 +884,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
audio_embeds = out_mm_data.get("audio_embeds")
if audio_embeds is not None:
embed = audio_embeds[item_idx]
assert isinstance(embed, torch.Tensor)
num_features = embed.shape[0]
else:
raise ValueError(
@@ -1118,7 +1125,11 @@ class GlmAsrForConditionalGeneration(
audio_token = cls._get_audio_token(model_config)
if task_type == "translate":
full_lang_name_to = cls.supported_languages.get(to_language, to_language)
full_lang_name_to = (
cls.supported_languages.get(to_language, to_language)
if to_language is not None
else ""
)
user_content = f"{audio_token}translate the speech to {full_lang_name_to}"
elif task_type == "transcribe":
user_content = (
+1
View File
@@ -227,6 +227,7 @@ class GPTJModel(nn.Module):
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.h, self.start_layer, self.end_layer):
hidden_states = layer(position_ids, hidden_states)
+1
View File
@@ -244,6 +244,7 @@ class GPTNeoXModel(nn.Module):
else:
hidden_states = self.embed_input_ids(input_ids)
else:
assert intermediate_tensors is not None
hidden_states = intermediate_tensors["hidden_states"]
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(position_ids, hidden_states)
+62 -21
View File
@@ -79,6 +79,13 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
}
def _get_weight_loader(param: torch.Tensor) -> Callable[..., object]:
weight_loader = getattr(param, "weight_loader", default_weight_loader)
if not callable(weight_loader):
raise TypeError("weight_loader must be callable")
return weight_loader
class OAIAttention(nn.Module):
# Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves);
# privacy-filter and similar derivatives use GPT-J (interleaved pairs).
@@ -273,6 +280,28 @@ class GptOssRoutedExperts(RoutedExperts):
self._copy_to_expert(expert_data, loaded_weight)
@typing.overload
def weight_loader(
self,
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: typing.Literal[False],
) -> None: ...
@typing.overload
def weight_loader(
self,
param: torch.nn.Parameter,
loaded_weight: torch.Tensor,
weight_name: str,
shard_id: str,
expert_id: int,
return_success: typing.Literal[True],
) -> bool: ...
def weight_loader(
self,
param: torch.nn.Parameter,
@@ -283,14 +312,24 @@ class GptOssRoutedExperts(RoutedExperts):
return_success: bool = False,
) -> bool | None:
if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"):
return super().weight_loader(
if return_success:
return super().weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
True,
)
super().weight_loader(
param,
loaded_weight,
weight_name,
shard_id,
expert_id,
return_success,
False,
)
return None
expert_id = self._map_global_expert_id_to_local_expert_id(expert_id)
if expert_id == -1:
@@ -518,7 +557,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
heads_per_rank: int,
head_start: int,
weights: Iterable[tuple[str, torch.Tensor]],
stacked_params_mapping: list[tuple[str, ...]],
stacked_params_mapping: list[tuple[str, str, str]],
) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -564,7 +603,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end, ...]
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(
param,
narrow_weight,
@@ -586,7 +625,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
]
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(
param,
narrow_weight,
@@ -612,7 +651,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end, ...]
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(
param,
narrow_weight,
@@ -635,7 +674,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
narrow_weight = weight[..., tp_rank_start // 2 : tp_rank_end // 2]
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(
param,
narrow_weight,
@@ -654,7 +693,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end]
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(
param,
narrow_weight,
@@ -667,7 +706,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
elif ".w2_bias" in name:
# Handle MLP down projection bias
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
if use_ep:
weight = weight[ep_rank_start:ep_rank_end, ...]
else:
@@ -691,7 +730,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
if weight_loader == default_weight_loader:
weight_loader(param, weight)
else:
@@ -702,7 +741,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(param, weight)
loaded_params.add(name)
return loaded_params
@@ -714,7 +753,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
heads_per_rank: int,
head_start: int,
weights: Iterable[tuple[str, torch.Tensor]],
stacked_params_mapping: list[tuple[str, ...]],
stacked_params_mapping: list[tuple[str, str, str]],
) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -820,6 +859,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
and expert_id is not None
):
assert loaded_weight.numel() == 1
assert fused_name is not None
expert_data = params_dict[fused_name].data[expert_id]
expert_data.copy_(loaded_weight)
loaded_params.add(fused_name)
@@ -901,6 +941,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
# fused gate_up_proj fused on disk, we cannot use the existing
# weight loaders without added complexity, so just do the
# direct load here.
assert fused_name is not None
param = params_dict[fused_name]
expert_data = param.data[expert_id]
dim1 = sliced_weight.shape[0]
@@ -1062,9 +1103,10 @@ class GptOssModel(nn.Module, EagleModelMixin):
if name.endswith("scale"):
# Remapping the name of FP8 kv-scale.
name = maybe_remap_kv_scale_name(name, params_dict)
if name is None:
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
if remapped_name is None:
continue
name = remapped_name
param = params_dict[name]
weight_loader = param.weight_loader
@@ -1084,6 +1126,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
if weight_name not in name:
continue
assert fused_name is not None
param = params_dict[fused_name]
# We should ask the weight loader to return success or not
# here since otherwise we may skip experts with other
@@ -1112,9 +1155,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(
param, "weight_loader", default_weight_loader
)
weight_loader = _get_weight_loader(param)
weight_loader(param, loaded_weight)
loaded_params.add(name)
@@ -1127,7 +1168,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
heads_per_rank: int,
head_start: int,
weights: Iterable[tuple[str, torch.Tensor]],
stacked_params_mapping: list[tuple[str, ...]],
stacked_params_mapping: list[tuple[str, str, str]],
) -> set[str]:
params_dict = dict(self.named_parameters())
loaded_params: set[str] = set()
@@ -1224,7 +1265,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
continue
name = name.replace(weight_name, param_name)
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
if weight_loader == default_weight_loader:
weight_loader(param, weight)
else:
@@ -1235,7 +1276,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
if name not in params_dict:
continue
param = params_dict[name]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = _get_weight_loader(param)
weight_loader(param, weight)
loaded_params.add(name)
return loaded_params
@@ -1367,7 +1408,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
class GptOssForCausalLM(
nn.Module, SupportsPP, SupportsEagle, SupportsEagle3, SupportsLoRA
):
is_3d_moe_weight: bool = True
is_3d_moe_weight: typing.ClassVar[bool] = True
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
hf_to_vllm_mapper = WeightsMapper(
@@ -130,6 +130,7 @@ class WindowQFormerDownsampler(nn.Module):
self.dropout = nn.Dropout(config.projector_dropout)
self.downsampler: SpatialOffsetDownsampler | InterpolateDownsampler
if spatial_offset is not None:
self.downsampler = SpatialOffsetDownsampler(config, offset=spatial_offset)
else:
@@ -843,6 +844,7 @@ class Granite4VisionForConditionalGeneration(
# (possibly a chunk slice from the framework's encoder cache).
# Concatenate along token dim → (total_mm_tokens, lm_h * num_levels).
N, lm_h = inputs_embeds.shape
assert multimodal_embeddings is not None
all_packed = torch.cat(
[t.to(dtype=inputs_embeds.dtype) for t in multimodal_embeddings],
dim=0,
+13 -3
View File
@@ -34,7 +34,7 @@ from torch import nn
from transformers import BatchFeature, PretrainedConfig
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
from vllm.config.multimodal import BaseDummyOptions
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
from vllm.config.speech_to_text import SpeechToTextParams
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
@@ -166,7 +166,9 @@ class GraniteSpeechMultiModalProcessor(
def get_replacement(item_idx: int):
audios = mm_items.get_items("audio", AudioProcessorItems)
audio = audios.get(item_idx)
audio_length = audio.shape[-1]
if audio is None:
raise ValueError(f"Missing audio item {item_idx}")
audio_length = len(audio) if isinstance(audio, list) else audio.shape[-1]
num_projector_features = feature_extractor._get_num_audio_features(
[audio_length]
)[0]
@@ -225,6 +227,7 @@ class GraniteSpeechDummyInputsBuilder(
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio")
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
return {
"audio": self._get_dummy_audios(
@@ -659,6 +662,9 @@ class GraniteSpeechForConditionalGeneration(
if input_features is None:
return None
if not isinstance(audio_embed_sizes, torch.Tensor):
raise ValueError("audio_embed_sizes must be a tensor")
# If we have a batch of variable feature length audio clips, we need
# to mask the features; usually we would get an input_features_mask
# from the processor, but we handle rebuilding it here since
@@ -879,7 +885,11 @@ class GraniteSpeechForConditionalGeneration(
audio_tok = cls.get_placeholder_str("audio", 0)
if task_type == "translate":
full_lang_name_to = cls.supported_languages.get(to_language, to_language)
full_lang_name_to = (
cls.supported_languages.get(to_language, to_language)
if to_language is not None
else ""
)
user_prompt = f"{audio_tok}translate the speech to {full_lang_name_to}" # noqa: E501
elif task_type == "transcribe":
user_prompt = (
@@ -270,6 +270,7 @@ class GraniteMoeHybridAttention(nn.Module):
prefix=f"{prefix}.o_proj",
)
self.rotary_emb: nn.Module | None
if config.position_embedding_type == "rope":
self.rotary_emb = get_rope(
self.head_dim,
@@ -454,7 +455,9 @@ class GraniteMoeHybridModel(nn.Module):
# Skip layers on other devices.
if not is_pp_missing_parameter(n, self):
param = params_dict[n]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = getattr(param, "weight_loader", None)
if not callable(weight_loader):
raise TypeError(f"{n} requires a shard-aware weight loader")
weight_loader(param, p, shard_id)
loaded_params.add(n)
@@ -462,7 +465,9 @@ class GraniteMoeHybridModel(nn.Module):
if n not in params_dict:
return
param = params_dict[n]
weight_loader = getattr(param, "weight_loader", default_weight_loader)
weight_loader = getattr(param, "weight_loader", None)
if not callable(weight_loader):
raise TypeError(f"{n} requires an expert-aware weight loader")
weight_loader(param, p, name, shard_id=shard_id, expert_id=expert_id)
loaded_params.add(n)