[Mypy Fix] Mypy fix for "vllm/model_executor/models/[gG]" (#53616)
Signed-off-by: yewentao256 <zhyanwentao@126.com>
This commit is contained in:
@@ -112,7 +112,6 @@ SEPARATE_GROUPS = [
|
||||
]
|
||||
|
||||
EXCLUDE = [
|
||||
r"vllm/model_executor/models/[gG]",
|
||||
r"vllm/model_executor/models/[hH]",
|
||||
r"vllm/model_executor/models/[iI]",
|
||||
r"vllm/model_executor/models/[jJ]",
|
||||
|
||||
@@ -457,7 +457,7 @@ class DeepseekV2Attention(nn.Module):
|
||||
qk_nope_head_dim: int,
|
||||
qk_rope_head_dim: int,
|
||||
v_head_dim: int,
|
||||
q_lora_rank: int,
|
||||
q_lora_rank: int | None,
|
||||
kv_lora_rank: int,
|
||||
max_position_embeddings: int = 8192,
|
||||
cache_config: CacheConfig | None = None,
|
||||
@@ -495,7 +495,7 @@ class DeepseekV2Attention(nn.Module):
|
||||
)
|
||||
self.q_a_layernorm = RMSNorm(self.q_lora_rank, eps=config.rms_norm_eps)
|
||||
self.q_b_proj = ColumnParallelLinear(
|
||||
q_lora_rank,
|
||||
self.q_lora_rank,
|
||||
self.num_heads * self.qk_head_dim,
|
||||
bias=False,
|
||||
quant_config=quant_config,
|
||||
|
||||
@@ -311,6 +311,7 @@ class GemmaModel(nn.Module):
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
residual = None
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
residual = intermediate_tensors["residual"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
|
||||
@@ -852,7 +852,9 @@ class Gemma3ForConditionalGeneration(
|
||||
def encoder_eager_forward(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
path: str = "default",
|
||||
) -> torch.Tensor:
|
||||
image_input = self._parse_and_validate_image_input(**mm_kwargs)
|
||||
assert image_input is not None
|
||||
results = self._process_image_input(image_input)
|
||||
return torch.cat(results, dim=0)
|
||||
|
||||
@@ -77,7 +77,7 @@ class Gemma3nAltUp(nn.Module):
|
||||
altup_num_inputs: int,
|
||||
altup_coef_clip: float,
|
||||
altup_active_idx: int,
|
||||
quant_config: QuantizationConfig,
|
||||
quant_config: QuantizationConfig | None,
|
||||
prefix: str,
|
||||
):
|
||||
super().__init__()
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
import inspect
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from collections.abc import Iterable, Mapping, Sequence, Sized
|
||||
from typing import Annotated, Any, Literal
|
||||
|
||||
import torch
|
||||
@@ -18,7 +18,7 @@ from transformers.models.gemma3n import (
|
||||
from transformers.models.siglip import SiglipImageProcessorFast
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.inputs import MultiModalDataDict, PromptType, TextPrompt
|
||||
from vllm.logger import init_logger
|
||||
@@ -115,8 +115,10 @@ def batch_audio_features(
|
||||
The `(bn, s_max, f)` features and their `(bn, s_max)` mask.
|
||||
"""
|
||||
if isinstance(input_features, torch.Tensor):
|
||||
assert isinstance(input_features_mask, torch.Tensor)
|
||||
return input_features.squeeze(1), input_features_mask.squeeze(1)
|
||||
|
||||
assert isinstance(input_features_mask, list)
|
||||
max_len = max(features.shape[0] for features in input_features)
|
||||
batched_features = input_features[0].new_zeros(
|
||||
(len(input_features), max_len, input_features[0].shape[-1])
|
||||
@@ -248,6 +250,7 @@ class Gemma3nDummyInputsBuilder(BaseDummyInputsBuilder[Gemma3nProcessingInfo]):
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
audio_overrides = mm_options.get("audio")
|
||||
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
@@ -293,10 +296,14 @@ class Gemma3nMultiModalProcessor(BaseMultiModalProcessor[Gemma3nProcessingInfo])
|
||||
# output (and thus its cache entry) is independent of the other
|
||||
# items in the batch. `batch_audio_features` re-pads the
|
||||
# per-item features so audio_tower can still run batched.
|
||||
num_frames = [
|
||||
self._get_num_audio_frames_alone(len(audio))
|
||||
for audio in mm_data["audio"]
|
||||
]
|
||||
audios = mm_data["audio"]
|
||||
if not isinstance(audios, Sequence):
|
||||
raise TypeError("audio data must be a sequence")
|
||||
num_frames = []
|
||||
for audio in audios:
|
||||
if not isinstance(audio, Sized):
|
||||
raise TypeError("each audio item must have a length")
|
||||
num_frames.append(self._get_num_audio_frames_alone(len(audio)))
|
||||
processed_data["input_features_padded"] = [
|
||||
features[:n]
|
||||
for features, n in zip(processed_data.pop("input_features"), num_frames)
|
||||
@@ -685,7 +692,9 @@ class Gemma3nForConditionalGeneration(
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[
|
||||
str, Gemma3nImageInputs | Gemma3nAudioInputs | None
|
||||
] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -852,7 +861,13 @@ class Gemma3nForConditionalGeneration(
|
||||
# select a chunk of pre-allocated PLEs. During normal execution,
|
||||
# `embed_input_ids` is called before forward, hence this slice
|
||||
# will contain PLEs computed from the actual input_ids.
|
||||
per_layer_inputs = self.per_layer_embeddings[: inputs_embeds.shape[0]]
|
||||
if inputs_embeds is not None:
|
||||
num_tokens = inputs_embeds.shape[0]
|
||||
elif intermediate_tensors is not None:
|
||||
num_tokens = intermediate_tensors["hidden_states"].shape[0]
|
||||
else:
|
||||
raise ValueError("inputs_embeds is required on the first PP rank")
|
||||
per_layer_inputs = self.per_layer_embeddings[:num_tokens]
|
||||
|
||||
hidden_states = self.language_model.model(
|
||||
input_ids,
|
||||
@@ -913,9 +928,15 @@ class Gemma3nForConditionalGeneration(
|
||||
prompt += " this audio"
|
||||
|
||||
# We assume the language is a valid ISO 639-1 code.
|
||||
full_lang_name = cls.supported_languages.get(language, "")
|
||||
full_lang_name = (
|
||||
cls.supported_languages.get(language, "") if language is not None else ""
|
||||
)
|
||||
# Translation only for now
|
||||
full_lang_name_to = cls.supported_languages.get(to_language, "")
|
||||
full_lang_name_to = (
|
||||
cls.supported_languages.get(to_language, "")
|
||||
if to_language is not None
|
||||
else ""
|
||||
)
|
||||
|
||||
if task_type == "transcribe" and full_lang_name:
|
||||
prompt += f" into {full_lang_name}"
|
||||
|
||||
@@ -352,15 +352,19 @@ class Gemma4MoE(nn.Module):
|
||||
)
|
||||
|
||||
# MoERunner experts with custom Gemma4 routing
|
||||
intermediate_size = getattr(
|
||||
config,
|
||||
"moe_intermediate_size",
|
||||
getattr(config, "expert_intermediate_size", None),
|
||||
)
|
||||
if intermediate_size is None:
|
||||
raise ValueError("Gemma4 MoE requires an expert intermediate size")
|
||||
|
||||
self.experts = FusedMoEFactory(
|
||||
num_experts=config.num_experts,
|
||||
top_k=config.top_k_experts,
|
||||
hidden_size=config.hidden_size,
|
||||
intermediate_size=getattr(
|
||||
config,
|
||||
"moe_intermediate_size",
|
||||
getattr(config, "expert_intermediate_size", None),
|
||||
),
|
||||
intermediate_size=intermediate_size,
|
||||
renormalize=True,
|
||||
quant_config=quant_config,
|
||||
prefix=f"{prefix}.experts",
|
||||
@@ -624,6 +628,11 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.enable_moe_block = getattr(config, "enable_moe_block", False) or getattr(
|
||||
config, "use_second_mlp_block", False
|
||||
)
|
||||
self.router: Gemma4Router | None
|
||||
self.moe: Gemma4MoE | None
|
||||
self.post_feedforward_layernorm_1: RMSNorm | None
|
||||
self.post_feedforward_layernorm_2: RMSNorm | None
|
||||
self.pre_feedforward_layernorm_2: RMSNorm | None
|
||||
if self.enable_moe_block:
|
||||
self.router = Gemma4Router(
|
||||
config,
|
||||
@@ -652,6 +661,9 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
self.pre_feedforward_layernorm_2 = None
|
||||
|
||||
# Per-Layer Embedding (PLE) components — present in each decoder layer
|
||||
self.per_layer_input_gate: ReplicatedLinear | None
|
||||
self.per_layer_projection: ReplicatedLinear | None
|
||||
self.post_per_layer_input_norm: RMSNorm | None
|
||||
if (
|
||||
self.hidden_size_per_layer_input is not None
|
||||
and self.hidden_size_per_layer_input > 0
|
||||
@@ -716,6 +728,11 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
hidden_states = self.mlp(hidden_states)
|
||||
|
||||
if self.enable_moe_block:
|
||||
assert self.post_feedforward_layernorm_1 is not None
|
||||
assert self.pre_feedforward_layernorm_2 is not None
|
||||
assert self.router is not None
|
||||
assert self.moe is not None
|
||||
assert self.post_feedforward_layernorm_2 is not None
|
||||
hidden_states_1 = self.post_feedforward_layernorm_1(hidden_states)
|
||||
|
||||
hidden_states_2 = self.pre_feedforward_layernorm_2(residual)
|
||||
@@ -731,6 +748,8 @@ class Gemma4DecoderLayer(nn.Module):
|
||||
|
||||
# Apply PLE (Per-Layer Embedding) if configured
|
||||
if per_layer_input is not None and self.per_layer_input_gate is not None:
|
||||
assert self.per_layer_projection is not None
|
||||
assert self.post_per_layer_input_norm is not None
|
||||
gate = self.per_layer_input_gate(hidden_states)
|
||||
gate = torch.nn.functional.gelu(gate, approximate="tanh")
|
||||
gated_per_layer = gate * per_layer_input
|
||||
@@ -865,6 +884,7 @@ class Gemma4SelfDecoderLayers(nn.Module):
|
||||
"""
|
||||
if self.per_layer_model_projection is None:
|
||||
return None
|
||||
assert self.per_layer_projection_norm is not None
|
||||
per_layer_projection = self.per_layer_model_projection(inputs_embeds)
|
||||
per_layer_projection = per_layer_projection * self.per_layer_projection_scale
|
||||
per_layer_projection = per_layer_projection.reshape(
|
||||
@@ -971,6 +991,9 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
)
|
||||
|
||||
# Per-Layer Embedding (PLE) components
|
||||
self.embed_tokens_per_layer: VocabParallelEmbedding | None
|
||||
self.per_layer_model_projection: ColumnParallelLinear | None
|
||||
self.per_layer_projection_norm: RMSNorm | None
|
||||
if (
|
||||
self.hidden_size_per_layer_input is not None
|
||||
and self.hidden_size_per_layer_input > 0
|
||||
@@ -1472,9 +1495,10 @@ class Gemma4Model(nn.Module, EagleModelMixin):
|
||||
else:
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
# Skip if name doesn't exist in params_dict (e.g., individual
|
||||
|
||||
@@ -89,7 +89,11 @@ class Gemma4DSparkAttention(Gemma4MTPAttention):
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
k, _ = self.k_proj(hidden_states)
|
||||
k_normed = self.k_norm(k.unflatten(-1, (self.num_kv_heads, self.head_dim)))
|
||||
v_src = k if self.use_k_eq_v else self.v_proj(hidden_states)[0]
|
||||
if self.use_k_eq_v:
|
||||
v_src = k
|
||||
else:
|
||||
assert self.v_proj is not None
|
||||
v_src = self.v_proj(hidden_states)[0]
|
||||
v_normed = self.v_norm(v_src.unflatten(-1, (self.num_kv_heads, self.head_dim)))
|
||||
return k_normed.flatten(-2, -1), v_normed.flatten(-2, -1)
|
||||
|
||||
|
||||
@@ -35,7 +35,11 @@ from transformers.models.gemma4.configuration_gemma4 import (
|
||||
|
||||
from vllm.config import VllmConfig
|
||||
from vllm.config.model import get_served_model_name
|
||||
from vllm.config.multimodal import BaseDummyOptions, VideoDummyOptions
|
||||
from vllm.config.multimodal import (
|
||||
AudioDummyOptions,
|
||||
BaseDummyOptions,
|
||||
VideoDummyOptions,
|
||||
)
|
||||
from vllm.inputs import MultiModalDataDict
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.layernorm import RMSNorm
|
||||
@@ -189,6 +193,7 @@ class Gemma4VideoInputs(TensorSchema):
|
||||
torch.Tensor,
|
||||
TensorShape("bn", "np", 2),
|
||||
]
|
||||
video_frame_counts: Annotated[torch.Tensor, TensorShape("nv")]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -490,8 +495,10 @@ class Gemma4DummyInputsBuilder(BaseDummyInputsBuilder[Gemma4ProcessingInfo]):
|
||||
image_overrides = mm_options.get("image") if mm_options else None
|
||||
audio_overrides = mm_options.get("audio") if mm_options else None
|
||||
video_overrides = mm_options.get("video") if mm_options else None
|
||||
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||
|
||||
data: MultiModalDataDict = {
|
||||
data: dict[str, Any] = {
|
||||
"image": self._get_dummy_images(
|
||||
width=img_width,
|
||||
height=img_height,
|
||||
@@ -592,6 +599,7 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
# same vision tower, matching transformers processing_gemma4.py.
|
||||
video_outputs: dict[str, Any] = {}
|
||||
if videos := mm_data.pop("videos", []):
|
||||
assert isinstance(videos, list)
|
||||
processor = self.info.get_hf_processor()
|
||||
|
||||
all_video_pixel_values: list[torch.Tensor] = []
|
||||
@@ -858,8 +866,29 @@ class Gemma4MultiModalProcessor(BaseMultiModalProcessor[Gemma4ProcessingInfo]):
|
||||
|
||||
def get_replacement_video(item_idx: int):
|
||||
out_item = out_mm_kwargs["video"][item_idx]
|
||||
timestamps = out_item["video_timestamps"].data
|
||||
num_soft = out_item["video_num_soft_tokens"].data
|
||||
timestamps_data = out_item["video_timestamps"].data
|
||||
if isinstance(timestamps_data, torch.Tensor):
|
||||
timestamps_data = timestamps_data.tolist()
|
||||
if not isinstance(timestamps_data, list):
|
||||
raise TypeError("video timestamps must be a list")
|
||||
timestamps: list[float] = []
|
||||
for timestamp in timestamps_data:
|
||||
if not isinstance(timestamp, (int, float)):
|
||||
raise TypeError("each video timestamp must be numeric")
|
||||
timestamps.append(float(timestamp))
|
||||
|
||||
num_soft_data = out_item["video_num_soft_tokens"].data
|
||||
if isinstance(num_soft_data, torch.Tensor):
|
||||
num_soft_data = num_soft_data.tolist()
|
||||
if not isinstance(num_soft_data, list):
|
||||
raise TypeError("video soft-token counts must be a list")
|
||||
num_soft: list[int] = []
|
||||
for count in num_soft_data:
|
||||
if not isinstance(count, int):
|
||||
raise TypeError(
|
||||
"each video soft-token count must be an integer"
|
||||
)
|
||||
num_soft.append(count)
|
||||
return self.info.get_video_repl(
|
||||
timestamps=timestamps,
|
||||
num_soft_tokens_per_frame=num_soft,
|
||||
@@ -1038,6 +1067,7 @@ class Gemma4ForConditionalGeneration(
|
||||
# (Marlin, FP8, …) require dimensions divisible by 64, which
|
||||
# the vision tower (intermediate_size=4304) does not satisfy.
|
||||
# TODO(mgoin): remove this by fixing kernel padding.
|
||||
tower_quant: QuantizationConfig | None
|
||||
if quant_config and quant_config.get_name() in [
|
||||
"bitsandbytes",
|
||||
"torchao",
|
||||
@@ -1068,6 +1098,7 @@ class Gemma4ForConditionalGeneration(
|
||||
)
|
||||
|
||||
# ---- Audio tower (variants with audio_config) ----
|
||||
self.embed_audio: Gemma4MultimodalEmbedder | None
|
||||
if config.audio_config is not None:
|
||||
with self._mark_tower_model(vllm_config, "audio"):
|
||||
self.audio_tower = AutoModel.from_config(config=config.audio_config)
|
||||
@@ -1177,22 +1208,24 @@ class Gemma4ForConditionalGeneration(
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
) -> dict[str, torch.Tensor] | None:
|
||||
) -> Gemma4VideoInputs | None:
|
||||
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
|
||||
pixel_position_ids_videos = kwargs.pop("pixel_position_ids_videos", None)
|
||||
video_frame_counts = kwargs.pop("video_frame_counts", None)
|
||||
if pixel_values_videos is None:
|
||||
return None
|
||||
return {
|
||||
"pixel_values_videos": pixel_values_videos,
|
||||
"pixel_position_ids_videos": pixel_position_ids_videos,
|
||||
"video_frame_counts": video_frame_counts,
|
||||
}
|
||||
return Gemma4VideoInputs(
|
||||
pixel_values_videos=pixel_values_videos,
|
||||
pixel_position_ids_videos=pixel_position_ids_videos,
|
||||
video_frame_counts=video_frame_counts,
|
||||
)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(
|
||||
self, **kwargs: object
|
||||
) -> dict[str, Gemma4ImageInputs | Gemma4AudioInputs | Gemma4VideoInputs | None]:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[
|
||||
str, Gemma4ImageInputs | Gemma4AudioInputs | Gemma4VideoInputs | None
|
||||
] = {}
|
||||
for input_key in list(kwargs):
|
||||
if (
|
||||
input_key in ("pixel_values", "image_embeds")
|
||||
@@ -1419,7 +1452,7 @@ class Gemma4ForConditionalGeneration(
|
||||
|
||||
def _process_video_input(
|
||||
self,
|
||||
video_input: dict[str, torch.Tensor],
|
||||
video_input: Gemma4VideoInputs,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Batch-encode video frames through the vision tower.
|
||||
|
||||
@@ -1543,6 +1576,8 @@ class Gemma4ForConditionalGeneration(
|
||||
audio_input["input_features_mask"],
|
||||
)
|
||||
|
||||
assert self.audio_tower is not None
|
||||
assert self.embed_audio is not None
|
||||
# Run audio tower — mask convention: True=valid, False=padding.
|
||||
audio_outputs = self.audio_tower(input_features, input_features_mask)
|
||||
if isinstance(audio_outputs, tuple):
|
||||
@@ -1576,14 +1611,17 @@ class Gemma4ForConditionalGeneration(
|
||||
if multimodal_input is None:
|
||||
continue
|
||||
if modality == "image":
|
||||
assert isinstance(multimodal_input, Gemma4ImageInputs)
|
||||
multimodal_embeddings.extend(
|
||||
self._process_image_input(multimodal_input)
|
||||
)
|
||||
elif modality == "video":
|
||||
assert isinstance(multimodal_input, Gemma4VideoInputs)
|
||||
multimodal_embeddings.extend(
|
||||
self._process_video_input(multimodal_input)
|
||||
)
|
||||
elif modality == "audio":
|
||||
assert isinstance(multimodal_input, Gemma4AudioInputs)
|
||||
multimodal_embeddings.extend(
|
||||
self._process_audio_input(multimodal_input)
|
||||
)
|
||||
@@ -2204,13 +2242,15 @@ class Gemma4ForConditionalGeneration(
|
||||
mm_kwargs: MultiModalKwargsItem | None,
|
||||
num_mm_embeds: int,
|
||||
) -> tuple[int, int | None]:
|
||||
tower_tokens: int | None = None
|
||||
connector_tokens: int | None = None
|
||||
if modality in ("image", "video"):
|
||||
vision_config = self.config.vision_config
|
||||
pooling_k2 = vision_config.pooling_kernel_size**2
|
||||
|
||||
if modality == "image":
|
||||
pixel_values_key = "pixel_values"
|
||||
max_soft_tokens = vision_config.default_output_length
|
||||
max_soft_tokens = int(vision_config.default_output_length)
|
||||
mm_processor_kwargs = getattr(
|
||||
getattr(self, "multimodal_config", None),
|
||||
"mm_processor_kwargs",
|
||||
@@ -2241,7 +2281,7 @@ class Gemma4ForConditionalGeneration(
|
||||
* pooling_k2
|
||||
)
|
||||
|
||||
if modality == "audio":
|
||||
elif modality == "audio":
|
||||
tower_tokens = num_mm_embeds
|
||||
connector_tokens = num_mm_embeds
|
||||
|
||||
@@ -2255,6 +2295,10 @@ class Gemma4ForConditionalGeneration(
|
||||
tower_tokens = audio_tokens
|
||||
connector_tokens = audio_tokens
|
||||
|
||||
else:
|
||||
raise ValueError(f"Unsupported modality: {modality}")
|
||||
|
||||
assert tower_tokens is not None
|
||||
return tower_tokens, connector_tokens
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -345,7 +345,9 @@ class Gemma4MultiTokenPredictor(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
config = speculative_config.draft_model_config.hf_config
|
||||
text_config = _get_text_config(config)
|
||||
quant_config = get_draft_quant_config(vllm_config)
|
||||
self.config = text_config
|
||||
@@ -467,7 +469,9 @@ class Gemma4MTP(nn.Module):
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
config = speculative_config.draft_model_config.hf_config
|
||||
text_config = _get_text_config(config)
|
||||
self.quant_config = get_draft_quant_config(vllm_config)
|
||||
self.config = config
|
||||
@@ -496,6 +500,7 @@ class Gemma4MTP(nn.Module):
|
||||
soft_cap=getattr(text_config, "final_logit_softcapping", None),
|
||||
)
|
||||
|
||||
self.masked_embedding: Gemma4MTPMaskedEmbedder | None
|
||||
if getattr(config, "use_ordered_embeddings", False):
|
||||
num_centroids = getattr(config, "num_centroids", 2048)
|
||||
top_k = getattr(config, "centroid_intermediate_top_k", 32)
|
||||
@@ -516,7 +521,7 @@ class Gemma4MTP(nn.Module):
|
||||
else:
|
||||
self.masked_embedding = None
|
||||
|
||||
draft_cfg = vllm_config.speculative_config.draft_model_config
|
||||
draft_cfg = speculative_config.draft_model_config
|
||||
gen_cfg = draft_cfg.try_get_generation_config()
|
||||
self._suppress_token_ids = gen_cfg.get("suppress_tokens") if gen_cfg else None
|
||||
|
||||
@@ -545,6 +550,7 @@ class Gemma4MTP(nn.Module):
|
||||
def _get_full_lm_head_weight(self) -> torch.Tensor:
|
||||
if self._stable_full_lm_head_weight is not None:
|
||||
return self._stable_full_lm_head_weight
|
||||
assert self.masked_embedding is not None
|
||||
lm_head_weight = self.lm_head.weight
|
||||
tp_size = get_tensor_model_parallel_world_size()
|
||||
if tp_size > 1:
|
||||
@@ -579,6 +585,7 @@ class Gemma4MTP(nn.Module):
|
||||
hidden_states: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
"""Sparse argmax via centroids masking. Returns token IDs directly."""
|
||||
assert self.masked_embedding is not None
|
||||
return self.masked_embedding.get_top_tokens(
|
||||
hidden_states,
|
||||
self._get_full_lm_head_weight(),
|
||||
|
||||
@@ -44,6 +44,7 @@ from vllm.model_executor.models.gemma4_mm import (
|
||||
Gemma4MultimodalEmbedder,
|
||||
Gemma4MultiModalProcessor,
|
||||
Gemma4ProcessingInfo,
|
||||
Gemma4VideoInputs,
|
||||
_get_max_soft_tokens,
|
||||
)
|
||||
from vllm.model_executor.models.module_mapping import MultiModelKeys
|
||||
@@ -265,22 +266,16 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
|
||||
self.audio_tower = None
|
||||
|
||||
# ---- Encoder-free vision embedder ----
|
||||
self.vision_embedder = (
|
||||
Gemma4UnifiedVisionEmbedder(
|
||||
config.vision_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "vision_embedder"),
|
||||
)
|
||||
if config.vision_config is not None
|
||||
else None
|
||||
if config.vision_config is None:
|
||||
raise ValueError("Gemma4 Unified requires a vision configuration")
|
||||
self.vision_embedder = Gemma4UnifiedVisionEmbedder(
|
||||
config.vision_config,
|
||||
quant_config=quant_config,
|
||||
prefix=maybe_prefix(prefix, "vision_embedder"),
|
||||
)
|
||||
self.embed_vision = (
|
||||
Gemma4MultimodalEmbedder(
|
||||
config.vision_config,
|
||||
config.text_config,
|
||||
)
|
||||
if config.vision_config is not None
|
||||
else None
|
||||
self.embed_vision = Gemma4MultimodalEmbedder(
|
||||
config.vision_config,
|
||||
config.text_config,
|
||||
)
|
||||
|
||||
# ---- Encoder-free audio embedder ----
|
||||
@@ -382,7 +377,7 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
|
||||
|
||||
def _process_video_input(
|
||||
self,
|
||||
video_input: dict[str, torch.Tensor],
|
||||
video_input: Gemma4VideoInputs,
|
||||
) -> list[torch.Tensor]:
|
||||
"""Project video frames to LM space, one frame at a time.
|
||||
|
||||
@@ -429,6 +424,7 @@ class Gemma4UnifiedForConditionalGeneration(Gemma4ForConditionalGeneration):
|
||||
audio_input["input_features_mask"],
|
||||
)
|
||||
|
||||
assert self.embed_audio is not None
|
||||
target_dtype = self.embed_audio.embedding_projection.weight.dtype
|
||||
audio_features = self.embed_audio(input_features.to(target_dtype))
|
||||
per_audio: list[torch.Tensor] = []
|
||||
|
||||
@@ -1393,9 +1393,15 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
|
||||
return list(timestamps_list)
|
||||
|
||||
def _get_video_frame_embed_token_id(self, hf_processor: object) -> int:
|
||||
if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA:
|
||||
return hf_processor.image_token_id
|
||||
return hf_processor.video_token_id
|
||||
attr = (
|
||||
"image_token_id"
|
||||
if isinstance(hf_processor, Glm4vProcessor) or TRANSFORMERS_WITH_GA
|
||||
else "video_token_id"
|
||||
)
|
||||
token_id = getattr(hf_processor, attr, None)
|
||||
if not isinstance(token_id, int):
|
||||
raise ValueError(f"Processor has no valid {attr}")
|
||||
return token_id
|
||||
|
||||
def _construct_video_placeholder(
|
||||
self,
|
||||
@@ -1479,6 +1485,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
|
||||
|
||||
image_overrides = mm_options.get("image")
|
||||
video_overrides = mm_options.get("video")
|
||||
assert video_overrides is None or isinstance(video_overrides, VideoDummyOptions)
|
||||
|
||||
return {
|
||||
"image": self._get_dummy_images(
|
||||
@@ -1519,7 +1526,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
|
||||
)
|
||||
videos = [v.copy() for v in videos]
|
||||
|
||||
video_items = []
|
||||
video_items: list[VideoItem] = []
|
||||
for video in videos:
|
||||
video_num_frames = video.shape[0]
|
||||
video_metadata = {
|
||||
@@ -1611,16 +1618,14 @@ class Glm4vMultiModalProcessor(BaseMultiModalProcessor[Glm4vProcessingInfo]):
|
||||
processed_data.update(passthrough_data)
|
||||
return processed_data
|
||||
|
||||
if (
|
||||
"videos" in mm_data
|
||||
and isinstance(mm_data["videos"], list)
|
||||
and len(mm_data["videos"]) > 0
|
||||
):
|
||||
videos = mm_data.get("videos")
|
||||
if isinstance(videos, list) and videos:
|
||||
video_grid_thw_lst = []
|
||||
pixel_values_videos_lst = []
|
||||
frame_embed_token_id = self.info._get_video_frame_embed_token_id(processor)
|
||||
swap_video_frame_tokens = frame_embed_token_id == processor.image_token_id
|
||||
for item in mm_data.pop("videos", []):
|
||||
mm_data.pop("videos")
|
||||
for item in videos:
|
||||
video_array, metadata = item
|
||||
|
||||
# don't update mm_kwargs inplace
|
||||
@@ -1794,6 +1799,7 @@ class Glm4vForConditionalGeneration(
|
||||
self.config = config
|
||||
self.model_config = vllm_config.model_config
|
||||
self.multimodal_config = multimodal_config
|
||||
assert multimodal_config is not None
|
||||
self.use_data_parallel = multimodal_config.mm_encoder_tp_mode == "data"
|
||||
self.is_multimodal_pruning_enabled = (
|
||||
multimodal_config.is_multimodal_pruning_enabled()
|
||||
@@ -1850,6 +1856,7 @@ class Glm4vForConditionalGeneration(
|
||||
image_embeds=image_embeds,
|
||||
image_grid_thw=image_grid_thw,
|
||||
)
|
||||
raise AssertionError("image input must contain pixels or embeddings")
|
||||
|
||||
def _parse_and_validate_video_input(
|
||||
self, **kwargs: object
|
||||
@@ -1874,6 +1881,7 @@ class Glm4vForConditionalGeneration(
|
||||
video_embeds=video_embeds,
|
||||
video_grid_thw=video_grid_thw,
|
||||
)
|
||||
raise AssertionError("video input must contain pixels or embeddings")
|
||||
|
||||
def _process_image_input(
|
||||
self, image_input: Glm4vImageInputs
|
||||
@@ -1996,12 +2004,12 @@ class Glm4vForConditionalGeneration(
|
||||
def _get_grid_thw_by_modality(
|
||||
self,
|
||||
mm_kwargs: dict[str, Any],
|
||||
) -> list[tuple[int, int, int]]:
|
||||
) -> list[list[int]]:
|
||||
grid_thw_key = f"{self.get_input_modality(mm_kwargs)}_grid_thw"
|
||||
grid_thw = mm_kwargs[grid_thw_key]
|
||||
if not isinstance(grid_thw, list):
|
||||
grid_thw = grid_thw.tolist()
|
||||
return grid_thw
|
||||
return [list(row) for row in grid_thw]
|
||||
|
||||
def get_encoder_cudagraph_item_specs(
|
||||
self,
|
||||
@@ -2188,7 +2196,7 @@ class Glm4vForConditionalGeneration(
|
||||
return self.visual(pixel_values, grid_thw)
|
||||
|
||||
def _parse_and_validate_multimodal_inputs(self, **kwargs: object) -> dict:
|
||||
mm_input_by_modality = {}
|
||||
mm_input_by_modality: dict[str, Glm4vImageInputs | Glm4vVideoInputs | None] = {}
|
||||
|
||||
# Preserve the order of modalities if there are multiple of them
|
||||
# from the order of kwargs.
|
||||
@@ -2223,9 +2231,17 @@ class Glm4vForConditionalGeneration(
|
||||
for modality in mm_input_by_modality:
|
||||
multimodal_input = mm_input_by_modality[modality]
|
||||
if modality == "image":
|
||||
assert isinstance(
|
||||
multimodal_input,
|
||||
(Glm4vImagePixelInputs, Glm4vImageEmbeddingInputs),
|
||||
)
|
||||
image_embeddings = self._process_image_input(multimodal_input)
|
||||
multimodal_embeddings += tuple(image_embeddings)
|
||||
if modality == "video":
|
||||
assert isinstance(
|
||||
multimodal_input,
|
||||
(Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs),
|
||||
)
|
||||
video_embeddings = self._process_video_input(multimodal_input)
|
||||
multimodal_embeddings += tuple(video_embeddings)
|
||||
return multimodal_embeddings
|
||||
@@ -2238,14 +2254,26 @@ class Glm4vForConditionalGeneration(
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
embed_ranges = mm_feature.mm_position.extract_embeds_range()
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
feature_data = mm_feature.data
|
||||
assert feature_data is not None
|
||||
grid_item = feature_data.get("image_grid_thw")
|
||||
assert grid_item is not None
|
||||
grid_data = grid_item.data
|
||||
assert isinstance(grid_data, torch.Tensor)
|
||||
t, h, w = grid_data.tolist()
|
||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||
assert len(embed_ranges) == 1
|
||||
offset, end = embed_ranges[0]
|
||||
assert end - offset + 1 == h * w // spatial_merge_size**2
|
||||
yield offset, t, h // spatial_merge_size, w // spatial_merge_size
|
||||
elif mm_feature.modality == "video":
|
||||
t, h, w = mm_feature.data["video_grid_thw"].data.tolist()
|
||||
feature_data = mm_feature.data
|
||||
assert feature_data is not None
|
||||
grid_item = feature_data.get("video_grid_thw")
|
||||
assert grid_item is not None
|
||||
grid_data = grid_item.data
|
||||
assert isinstance(grid_data, torch.Tensor)
|
||||
t, h, w = grid_data.tolist()
|
||||
llm_grid_h = h // spatial_merge_size
|
||||
llm_grid_w = w // spatial_merge_size
|
||||
num_tokens_per_frame = llm_grid_h * llm_grid_w
|
||||
|
||||
@@ -502,6 +502,8 @@ class Glm4MoeModel(nn.Module):
|
||||
|
||||
|
||||
class Glm4MixtureOfExperts(MixtureOfExperts):
|
||||
moe_mlp_layers: list[Glm4MoE]
|
||||
|
||||
def extract_moe_parameters(self, example_moe: Glm4MoE | None) -> None:
|
||||
if example_moe is None:
|
||||
raise RuntimeError("No Glm4MoE layer found in model.layers.")
|
||||
|
||||
@@ -74,7 +74,6 @@ from .utils import (
|
||||
PPMissingLayer,
|
||||
get_spec_layer_idx_from_weight_name,
|
||||
is_pp_missing_parameter,
|
||||
make_empty_intermediate_tensors_factory,
|
||||
make_layers,
|
||||
maybe_prefix,
|
||||
)
|
||||
@@ -132,6 +131,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
v_head_dim = getattr(config, "v_head_dim", 0)
|
||||
kv_lora_rank = getattr(config, "kv_lora_rank", 0)
|
||||
|
||||
attn_cls: Callable[..., Glm4MoeLiteMLAAttention | Glm4MoeLiteAttention]
|
||||
if model_config.use_mla:
|
||||
attn_cls = Glm4MoeLiteMLAAttention
|
||||
else:
|
||||
@@ -145,7 +145,7 @@ class Glm4MoeLiteDecoderLayer(nn.Module):
|
||||
qk_nope_head_dim=qk_nope_head_dim,
|
||||
qk_rope_head_dim=qk_rope_head_dim,
|
||||
v_head_dim=v_head_dim,
|
||||
q_lora_rank=config.q_lora_rank if hasattr(config, "q_lora_rank") else None,
|
||||
q_lora_rank=getattr(config, "q_lora_rank", None),
|
||||
kv_lora_rank=kv_lora_rank,
|
||||
max_position_embeddings=max_position_embeddings,
|
||||
cache_config=cache_config,
|
||||
@@ -266,9 +266,6 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
|
||||
else:
|
||||
self.norm = PPMissingLayer()
|
||||
self.make_empty_intermediate_tensors = make_empty_intermediate_tensors_factory(
|
||||
["hidden_states", "residual"], config.hidden_size
|
||||
)
|
||||
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.embed_tokens(input_ids)
|
||||
@@ -454,7 +451,9 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
# param and delegate to its expert-aware weight_loader
|
||||
# with expert_id.
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
param_name, weight_name, mapping_expert_id, mapping_shard_id = (
|
||||
mapping
|
||||
)
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
|
||||
@@ -480,8 +479,8 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
shard_id=mapping_shard_id,
|
||||
expert_id=mapping_expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
@@ -502,9 +501,10 @@ class Glm4MoeLiteModel(nn.Module):
|
||||
continue
|
||||
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
if is_pp_missing_parameter(name, self):
|
||||
continue
|
||||
|
||||
@@ -85,7 +85,9 @@ class Glm4MoeLiteMultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__()
|
||||
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
config = speculative_config.draft_model_config.hf_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
@@ -235,7 +237,7 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
@@ -370,7 +372,9 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
# with expert_id.
|
||||
is_expert_weight = False
|
||||
for mapping in expert_params_mapping:
|
||||
param_name, weight_name, expert_id, shard_id = mapping
|
||||
param_name, weight_name, mapping_expert_id, mapping_shard_id = (
|
||||
mapping
|
||||
)
|
||||
if weight_name not in chunk_name:
|
||||
continue
|
||||
|
||||
@@ -393,8 +397,8 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
param,
|
||||
weight_to_load,
|
||||
name_mapped,
|
||||
shard_id=shard_id,
|
||||
expert_id=expert_id,
|
||||
shard_id=mapping_shard_id,
|
||||
expert_id=mapping_expert_id,
|
||||
return_success=True,
|
||||
)
|
||||
if success:
|
||||
@@ -414,9 +418,10 @@ class Glm4MoeLiteMTP(nn.Module, SupportsPP, Glm4MixtureOfExperts):
|
||||
if name.endswith(".bias") and name not in params_dict:
|
||||
continue
|
||||
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
# According to DeepSeek-V3 Technical Report, MTP modules
|
||||
# shares embedding layer. We only load the first weights.
|
||||
|
||||
@@ -93,6 +93,7 @@ class Glm4MoeMultiTokenPredictorLayer(nn.Module):
|
||||
self.shared_head = SharedHead(
|
||||
config=config, prefix=prefix, quant_config=quant_config
|
||||
)
|
||||
assert parallel_config is not None
|
||||
self.enable_eplb = parallel_config.enable_eplb
|
||||
self.mtp_block = Glm4MoeDecoderLayer(
|
||||
config=config,
|
||||
|
||||
@@ -582,7 +582,13 @@ class GLM4VForCausalLM(
|
||||
for mm_feature in sorted(mm_features, key=lambda f: f.mm_position.offset):
|
||||
offset = mm_feature.mm_position.offset
|
||||
if mm_feature.modality == "image":
|
||||
t, h, w = mm_feature.data["image_grid_thw"].data.tolist()
|
||||
feature_data = mm_feature.data
|
||||
assert feature_data is not None
|
||||
grid_item = feature_data.get("image_grid_thw")
|
||||
assert grid_item is not None
|
||||
grid_data = grid_item.data
|
||||
assert isinstance(grid_data, torch.Tensor)
|
||||
t, h, w = grid_data.tolist()
|
||||
assert t == 1, f"Image must have 1 frame, got {t}"
|
||||
yield offset, t, h // spatial_merge_size, w // spatial_merge_size
|
||||
else:
|
||||
@@ -625,7 +631,18 @@ class GLM4VForCausalLM(
|
||||
mrope_position_delta = (llm_positions.max() + 1 - len(input_tokens)).item()
|
||||
return torch.from_numpy(llm_positions), mrope_position_delta
|
||||
|
||||
embed_input_ids = SupportsMultiModal.embed_input_ids
|
||||
def embed_input_ids(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
return SupportsMultiModal.embed_input_ids(
|
||||
self,
|
||||
input_ids,
|
||||
multimodal_embeddings,
|
||||
is_multimodal=is_multimodal,
|
||||
)
|
||||
|
||||
def embed_multimodal(self, **kwargs: object) -> MultiModalEmbeddings:
|
||||
image_input = self._parse_and_validate_image_input(**kwargs)
|
||||
|
||||
@@ -326,7 +326,9 @@ class GlmOcrVisionTransformer(Glm4vVisionTransformer):
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
grid_thw: torch.Tensor | list[list[int]],
|
||||
encoder_metadata: dict[str, torch.Tensor] | None = None,
|
||||
) -> torch.Tensor:
|
||||
del encoder_metadata
|
||||
if isinstance(grid_thw, list):
|
||||
grid_thw = torch.tensor(grid_thw, dtype=torch.int32)
|
||||
|
||||
|
||||
@@ -58,7 +58,9 @@ class GlmOcrMultiTokenPredictorLayer(nn.Module):
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
nn.Module.__init__(self)
|
||||
|
||||
config = vllm_config.speculative_config.draft_model_config.hf_config.text_config
|
||||
speculative_config = vllm_config.speculative_config
|
||||
assert speculative_config is not None
|
||||
config = speculative_config.draft_model_config.hf_config.text_config
|
||||
self.config = config
|
||||
quant_config = vllm_config.quant_config
|
||||
|
||||
@@ -144,7 +146,7 @@ class GlmOcrMTP(nn.Module, SupportsPP):
|
||||
def embed_input_ids(self, input_ids: torch.Tensor) -> torch.Tensor:
|
||||
return self.model.embed_input_ids(input_ids)
|
||||
|
||||
def forward(
|
||||
def forward( # type: ignore[override]
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
positions: torch.Tensor,
|
||||
@@ -195,9 +197,10 @@ class GlmOcrMTP(nn.Module, SupportsPP):
|
||||
|
||||
if "scale" in name or "zero_point" in name:
|
||||
# Remapping the name of FP8 kv-scale or zero point.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
for param_name, weight_name, shard_id in stacked_params_mapping:
|
||||
if weight_name not in name:
|
||||
|
||||
@@ -11,7 +11,7 @@ from transformers.models.glmasr import GlmAsrConfig, GlmAsrProcessor
|
||||
from transformers.models.whisper import WhisperFeatureExtractor
|
||||
|
||||
from vllm.config import ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.distributed.parallel_state import get_tensor_model_parallel_world_size
|
||||
from vllm.inputs import ModalityData, MultiModalDataDict, PromptType, TokensPrompt
|
||||
@@ -710,6 +710,7 @@ class GlmAsrDummyInputsBuilder(BaseDummyInputsBuilder[GlmAsrProcessingInfo]):
|
||||
sampling_rate = feature_extractor.sampling_rate
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
audio_overrides = mm_options.get("audio")
|
||||
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||
|
||||
max_audio_len = getattr(
|
||||
self.info.get_hf_processor(), "max_audio_len", DEFAULT_MAX_AUDIO_LEN_S
|
||||
@@ -793,8 +794,13 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
)
|
||||
processed_data["feature_attention_mask"] = mask
|
||||
|
||||
audio = mm_data.get("audio", [])
|
||||
audio_list = [audio] if audio and not isinstance(audio, list) else audio
|
||||
audio = mm_data.get("audio")
|
||||
if audio is None:
|
||||
audio_list: list[Any] = []
|
||||
elif isinstance(audio, list):
|
||||
audio_list = audio
|
||||
else:
|
||||
audio_list = [audio]
|
||||
if audio_list:
|
||||
processor = self.info.get_hf_processor(**hf_processor_mm_kwargs)
|
||||
|
||||
@@ -878,6 +884,7 @@ class GlmAsrMultiModalProcessor(BaseMultiModalProcessor["GlmAsrProcessingInfo"])
|
||||
audio_embeds = out_mm_data.get("audio_embeds")
|
||||
if audio_embeds is not None:
|
||||
embed = audio_embeds[item_idx]
|
||||
assert isinstance(embed, torch.Tensor)
|
||||
num_features = embed.shape[0]
|
||||
else:
|
||||
raise ValueError(
|
||||
@@ -1118,7 +1125,11 @@ class GlmAsrForConditionalGeneration(
|
||||
audio_token = cls._get_audio_token(model_config)
|
||||
|
||||
if task_type == "translate":
|
||||
full_lang_name_to = cls.supported_languages.get(to_language, to_language)
|
||||
full_lang_name_to = (
|
||||
cls.supported_languages.get(to_language, to_language)
|
||||
if to_language is not None
|
||||
else ""
|
||||
)
|
||||
user_content = f"{audio_token}translate the speech to {full_lang_name_to}"
|
||||
elif task_type == "transcribe":
|
||||
user_content = (
|
||||
|
||||
@@ -227,6 +227,7 @@ class GPTJModel(nn.Module):
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.h, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(position_ids, hidden_states)
|
||||
|
||||
@@ -244,6 +244,7 @@ class GPTNeoXModel(nn.Module):
|
||||
else:
|
||||
hidden_states = self.embed_input_ids(input_ids)
|
||||
else:
|
||||
assert intermediate_tensors is not None
|
||||
hidden_states = intermediate_tensors["hidden_states"]
|
||||
for layer in islice(self.layers, self.start_layer, self.end_layer):
|
||||
hidden_states = layer(position_ids, hidden_states)
|
||||
|
||||
@@ -79,6 +79,13 @@ _GPT_OSS_STREAMED_EXPERT_SUFFIX_TO_SHARD = {
|
||||
}
|
||||
|
||||
|
||||
def _get_weight_loader(param: torch.Tensor) -> Callable[..., object]:
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
if not callable(weight_loader):
|
||||
raise TypeError("weight_loader must be callable")
|
||||
return weight_loader
|
||||
|
||||
|
||||
class OAIAttention(nn.Module):
|
||||
# Override to switch RoPE convention. gpt-oss uses NeoX (chunk halves);
|
||||
# privacy-filter and similar derivatives use GPT-J (interleaved pairs).
|
||||
@@ -273,6 +280,28 @@ class GptOssRoutedExperts(RoutedExperts):
|
||||
|
||||
self._copy_to_expert(expert_data, loaded_weight)
|
||||
|
||||
@typing.overload
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: typing.Literal[False],
|
||||
) -> None: ...
|
||||
|
||||
@typing.overload
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.nn.Parameter,
|
||||
loaded_weight: torch.Tensor,
|
||||
weight_name: str,
|
||||
shard_id: str,
|
||||
expert_id: int,
|
||||
return_success: typing.Literal[True],
|
||||
) -> bool: ...
|
||||
|
||||
def weight_loader(
|
||||
self,
|
||||
param: torch.nn.Parameter,
|
||||
@@ -283,14 +312,24 @@ class GptOssRoutedExperts(RoutedExperts):
|
||||
return_success: bool = False,
|
||||
) -> bool | None:
|
||||
if shard_id not in ("gpt_oss_w13", "gpt_oss_w2"):
|
||||
return super().weight_loader(
|
||||
if return_success:
|
||||
return super().weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
True,
|
||||
)
|
||||
super().weight_loader(
|
||||
param,
|
||||
loaded_weight,
|
||||
weight_name,
|
||||
shard_id,
|
||||
expert_id,
|
||||
return_success,
|
||||
False,
|
||||
)
|
||||
return None
|
||||
|
||||
expert_id = self._map_global_expert_id_to_local_expert_id(expert_id)
|
||||
if expert_id == -1:
|
||||
@@ -518,7 +557,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
heads_per_rank: int,
|
||||
head_start: int,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
stacked_params_mapping: list[tuple[str, ...]],
|
||||
stacked_params_mapping: list[tuple[str, str, str]],
|
||||
) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -564,7 +603,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end, ...]
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(
|
||||
param,
|
||||
narrow_weight,
|
||||
@@ -586,7 +625,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
]
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(
|
||||
param,
|
||||
narrow_weight,
|
||||
@@ -612,7 +651,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end, ...]
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(
|
||||
param,
|
||||
narrow_weight,
|
||||
@@ -635,7 +674,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
narrow_weight = weight[..., tp_rank_start // 2 : tp_rank_end // 2]
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(
|
||||
param,
|
||||
narrow_weight,
|
||||
@@ -654,7 +693,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
narrow_weight = weight[:, 2 * tp_rank_start : 2 * tp_rank_end]
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(
|
||||
param,
|
||||
narrow_weight,
|
||||
@@ -667,7 +706,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
elif ".w2_bias" in name:
|
||||
# Handle MLP down projection bias
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
if use_ep:
|
||||
weight = weight[ep_rank_start:ep_rank_end, ...]
|
||||
else:
|
||||
@@ -691,7 +730,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, weight)
|
||||
else:
|
||||
@@ -702,7 +741,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(param, weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
@@ -714,7 +753,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
heads_per_rank: int,
|
||||
head_start: int,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
stacked_params_mapping: list[tuple[str, ...]],
|
||||
stacked_params_mapping: list[tuple[str, str, str]],
|
||||
) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -820,6 +859,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
and expert_id is not None
|
||||
):
|
||||
assert loaded_weight.numel() == 1
|
||||
assert fused_name is not None
|
||||
expert_data = params_dict[fused_name].data[expert_id]
|
||||
expert_data.copy_(loaded_weight)
|
||||
loaded_params.add(fused_name)
|
||||
@@ -901,6 +941,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
# fused gate_up_proj fused on disk, we cannot use the existing
|
||||
# weight loaders without added complexity, so just do the
|
||||
# direct load here.
|
||||
assert fused_name is not None
|
||||
param = params_dict[fused_name]
|
||||
expert_data = param.data[expert_id]
|
||||
dim1 = sliced_weight.shape[0]
|
||||
@@ -1062,9 +1103,10 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
|
||||
if name.endswith("scale"):
|
||||
# Remapping the name of FP8 kv-scale.
|
||||
name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if name is None:
|
||||
remapped_name = maybe_remap_kv_scale_name(name, params_dict)
|
||||
if remapped_name is None:
|
||||
continue
|
||||
name = remapped_name
|
||||
|
||||
param = params_dict[name]
|
||||
weight_loader = param.weight_loader
|
||||
@@ -1084,6 +1126,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
if weight_name not in name:
|
||||
continue
|
||||
|
||||
assert fused_name is not None
|
||||
param = params_dict[fused_name]
|
||||
# We should ask the weight loader to return success or not
|
||||
# here since otherwise we may skip experts with other
|
||||
@@ -1112,9 +1155,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(
|
||||
param, "weight_loader", default_weight_loader
|
||||
)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(param, loaded_weight)
|
||||
|
||||
loaded_params.add(name)
|
||||
@@ -1127,7 +1168,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
heads_per_rank: int,
|
||||
head_start: int,
|
||||
weights: Iterable[tuple[str, torch.Tensor]],
|
||||
stacked_params_mapping: list[tuple[str, ...]],
|
||||
stacked_params_mapping: list[tuple[str, str, str]],
|
||||
) -> set[str]:
|
||||
params_dict = dict(self.named_parameters())
|
||||
loaded_params: set[str] = set()
|
||||
@@ -1224,7 +1265,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
continue
|
||||
name = name.replace(weight_name, param_name)
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
if weight_loader == default_weight_loader:
|
||||
weight_loader(param, weight)
|
||||
else:
|
||||
@@ -1235,7 +1276,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
if name not in params_dict:
|
||||
continue
|
||||
param = params_dict[name]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = _get_weight_loader(param)
|
||||
weight_loader(param, weight)
|
||||
loaded_params.add(name)
|
||||
return loaded_params
|
||||
@@ -1367,7 +1408,7 @@ class GptOssModel(nn.Module, EagleModelMixin):
|
||||
class GptOssForCausalLM(
|
||||
nn.Module, SupportsPP, SupportsEagle, SupportsEagle3, SupportsLoRA
|
||||
):
|
||||
is_3d_moe_weight: bool = True
|
||||
is_3d_moe_weight: typing.ClassVar[bool] = True
|
||||
packed_modules_mapping = {"qkv_proj": ["q_proj", "k_proj", "v_proj"]}
|
||||
|
||||
hf_to_vllm_mapper = WeightsMapper(
|
||||
|
||||
@@ -130,6 +130,7 @@ class WindowQFormerDownsampler(nn.Module):
|
||||
|
||||
self.dropout = nn.Dropout(config.projector_dropout)
|
||||
|
||||
self.downsampler: SpatialOffsetDownsampler | InterpolateDownsampler
|
||||
if spatial_offset is not None:
|
||||
self.downsampler = SpatialOffsetDownsampler(config, offset=spatial_offset)
|
||||
else:
|
||||
@@ -843,6 +844,7 @@ class Granite4VisionForConditionalGeneration(
|
||||
# (possibly a chunk slice from the framework's encoder cache).
|
||||
# Concatenate along token dim → (total_mm_tokens, lm_h * num_levels).
|
||||
N, lm_h = inputs_embeds.shape
|
||||
assert multimodal_embeddings is not None
|
||||
all_packed = torch.cat(
|
||||
[t.to(dtype=inputs_embeds.dtype) for t in multimodal_embeddings],
|
||||
dim=0,
|
||||
|
||||
@@ -34,7 +34,7 @@ from torch import nn
|
||||
from transformers import BatchFeature, PretrainedConfig
|
||||
|
||||
from vllm.config import CacheConfig, ModelConfig, SpeechToTextConfig, VllmConfig
|
||||
from vllm.config.multimodal import BaseDummyOptions
|
||||
from vllm.config.multimodal import AudioDummyOptions, BaseDummyOptions
|
||||
from vllm.config.speech_to_text import SpeechToTextParams
|
||||
from vllm.inputs import MultiModalDataDict, PromptType, TokensPrompt
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
@@ -166,7 +166,9 @@ class GraniteSpeechMultiModalProcessor(
|
||||
def get_replacement(item_idx: int):
|
||||
audios = mm_items.get_items("audio", AudioProcessorItems)
|
||||
audio = audios.get(item_idx)
|
||||
audio_length = audio.shape[-1]
|
||||
if audio is None:
|
||||
raise ValueError(f"Missing audio item {item_idx}")
|
||||
audio_length = len(audio) if isinstance(audio, list) else audio.shape[-1]
|
||||
num_projector_features = feature_extractor._get_num_audio_features(
|
||||
[audio_length]
|
||||
)[0]
|
||||
@@ -225,6 +227,7 @@ class GraniteSpeechDummyInputsBuilder(
|
||||
) -> MultiModalDataDict:
|
||||
num_audios = mm_counts.get("audio", 0)
|
||||
audio_overrides = mm_options.get("audio")
|
||||
assert audio_overrides is None or isinstance(audio_overrides, AudioDummyOptions)
|
||||
|
||||
return {
|
||||
"audio": self._get_dummy_audios(
|
||||
@@ -659,6 +662,9 @@ class GraniteSpeechForConditionalGeneration(
|
||||
if input_features is None:
|
||||
return None
|
||||
|
||||
if not isinstance(audio_embed_sizes, torch.Tensor):
|
||||
raise ValueError("audio_embed_sizes must be a tensor")
|
||||
|
||||
# If we have a batch of variable feature length audio clips, we need
|
||||
# to mask the features; usually we would get an input_features_mask
|
||||
# from the processor, but we handle rebuilding it here since
|
||||
@@ -879,7 +885,11 @@ class GraniteSpeechForConditionalGeneration(
|
||||
audio_tok = cls.get_placeholder_str("audio", 0)
|
||||
|
||||
if task_type == "translate":
|
||||
full_lang_name_to = cls.supported_languages.get(to_language, to_language)
|
||||
full_lang_name_to = (
|
||||
cls.supported_languages.get(to_language, to_language)
|
||||
if to_language is not None
|
||||
else ""
|
||||
)
|
||||
user_prompt = f"{audio_tok}translate the speech to {full_lang_name_to}" # noqa: E501
|
||||
elif task_type == "transcribe":
|
||||
user_prompt = (
|
||||
|
||||
@@ -270,6 +270,7 @@ class GraniteMoeHybridAttention(nn.Module):
|
||||
prefix=f"{prefix}.o_proj",
|
||||
)
|
||||
|
||||
self.rotary_emb: nn.Module | None
|
||||
if config.position_embedding_type == "rope":
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
@@ -454,7 +455,9 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
# Skip layers on other devices.
|
||||
if not is_pp_missing_parameter(n, self):
|
||||
param = params_dict[n]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = getattr(param, "weight_loader", None)
|
||||
if not callable(weight_loader):
|
||||
raise TypeError(f"{n} requires a shard-aware weight loader")
|
||||
weight_loader(param, p, shard_id)
|
||||
loaded_params.add(n)
|
||||
|
||||
@@ -462,7 +465,9 @@ class GraniteMoeHybridModel(nn.Module):
|
||||
if n not in params_dict:
|
||||
return
|
||||
param = params_dict[n]
|
||||
weight_loader = getattr(param, "weight_loader", default_weight_loader)
|
||||
weight_loader = getattr(param, "weight_loader", None)
|
||||
if not callable(weight_loader):
|
||||
raise TypeError(f"{n} requires an expert-aware weight loader")
|
||||
weight_loader(param, p, name, shard_id=shard_id, expert_id=expert_id)
|
||||
loaded_params.add(n)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user