Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-12 09:51:31 -07:00
committed by GitHub
parent 9bb38130cb
commit 8fcaaf6a16
944 changed files with 9490 additions and 10121 deletions
+4 -4
View File
@@ -4,7 +4,7 @@
import ast
import inspect
from collections.abc import Iterable
from typing import TYPE_CHECKING, Any, Optional, TypeVar, cast
from typing import TYPE_CHECKING, Any, TypeVar, cast
import torch
import torch.nn as nn
@@ -32,7 +32,7 @@ _GENERATE_SUFFIXES = [
]
def _load_st_projector(model_config: "ModelConfig") -> Optional[nn.Module]:
def _load_st_projector(model_config: "ModelConfig") -> nn.Module | None:
"""Load Sentence-Transformers Dense projection layers."""
try:
@@ -357,8 +357,8 @@ def as_seq_cls_model(cls: _T) -> _T:
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
return super().forward(
input_ids, positions, intermediate_tensors, inputs_embeds
+2 -3
View File
@@ -4,7 +4,6 @@
# A modified implementation of the AIMv2 Transformer
# inserted here also the image tokenizer used by Ovis2
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -165,7 +164,7 @@ class AIMv2Transformer(nn.Module):
config: AIMv2Config,
quant_config: QuantizationConfig,
*,
require_post_norm: Optional[bool] = None,
require_post_norm: bool | None = None,
prefix: str = "",
):
super().__init__()
@@ -196,7 +195,7 @@ class AIMv2Model(torch.nn.Module):
config: AIMv2Config,
quant_config: QuantizationConfig,
*,
require_post_norm: Optional[bool] = None,
require_post_norm: bool | None = None,
prefix: str = "",
):
super().__init__()
+18 -20
View File
@@ -27,7 +27,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -77,7 +77,7 @@ class ApertusMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
reduce_results: bool = True,
@@ -120,12 +120,12 @@ class ApertusAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
bias_o_proj: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
@@ -225,8 +225,8 @@ class ApertusAttention(nn.Module):
def _init_rotary_emb(
self,
config: ApertusConfig,
rope_scaling: Optional[dict[str, Any]],
quant_config: Optional[QuantizationConfig],
rope_scaling: dict[str, Any] | None,
quant_config: QuantizationConfig | None,
) -> None:
is_neox_style = True
is_gguf = quant_config and quant_config.get_name() == "gguf"
@@ -248,8 +248,8 @@ class ApertusDecoderLayer(nn.Module):
def __init__(
self,
config: ApertusConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -316,7 +316,7 @@ class ApertusDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -394,13 +394,11 @@ class ApertusModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[
torch.Tensor, IntermediateTensors, tuple[torch.Tensor, list[torch.Tensor]]
]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -586,9 +584,9 @@ class ApertusForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -597,7 +595,7 @@ class ApertusForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+13 -15
View File
@@ -10,7 +10,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -52,7 +52,7 @@ class ArceeMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[Any] = None,
quant_config: Any | None = None,
bias: bool = False,
prefix: str = "",
reduce_results: bool = True,
@@ -98,8 +98,8 @@ class ArceeDecoderLayer(nn.Module):
def __init__(
self,
config: LlamaConfig,
cache_config: Optional[Any] = None,
quant_config: Optional[Any] = None,
cache_config: Any | None = None,
quant_config: Any | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -165,7 +165,7 @@ class ArceeDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self-Attention block
if residual is None:
@@ -247,13 +247,11 @@ class ArceeModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[
torch.Tensor, IntermediateTensors, tuple[torch.Tensor, list[torch.Tensor]]
]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
# Embedding lookup (on first pipeline rank)
if get_pp_group().is_first_rank:
hidden_states = (
@@ -415,9 +413,9 @@ class ArceeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids=input_ids,
positions=positions,
@@ -426,7 +424,7 @@ class ArceeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
)
return model_output
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
# Compute final logits from hidden states (last pipeline rank only)
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+15 -16
View File
@@ -4,7 +4,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -63,7 +62,7 @@ class ArcticMLP(nn.Module):
config: ArcticConfig,
expert_id: int = -1,
is_residual_mlp: bool = False,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
):
@@ -107,9 +106,9 @@ class ArcticMoE(nn.Module):
def __init__(
self,
config: ArcticConfig,
tp_size: Optional[int] = None,
params_dtype: Optional[torch.dtype] = None,
quant_config: Optional[QuantizationConfig] = None,
tp_size: int | None = None,
params_dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
):
@@ -265,8 +264,8 @@ class ArcticAttention(nn.Module):
def __init__(
self,
config: ArcticConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -342,8 +341,8 @@ class ArcticDecoderLayer(nn.Module):
def __init__(
self,
config: ArcticConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -443,9 +442,9 @@ class ArcticModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -499,9 +498,9 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -510,7 +509,7 @@ class ArcticForCausalLM(nn.Module, SupportsPP, SupportsQuant):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+14 -14
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal
import torch
import torch.nn as nn
@@ -71,7 +71,7 @@ class AriaImagePixelInputs(TensorSchema):
]
pixel_mask: Annotated[
Optional[torch.Tensor],
torch.Tensor | None,
TensorShape("bn", "h", "w"),
]
@@ -82,7 +82,7 @@ class AriaVisionTransformer(Idefics3VisionTransformer, SupportsQuant):
def __init__(
self,
config: Idefics2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__(config, quant_config=quant_config, prefix=prefix)
@@ -180,7 +180,7 @@ class AriaProjector(nn.Module):
def forward(
self,
x: torch.Tensor,
attn_mask: Optional[torch.Tensor] = None,
attn_mask: torch.Tensor | None = None,
) -> torch.Tensor:
batch_size, num_patches = x.shape[0], x.shape[1]
@@ -250,7 +250,7 @@ class AriaTextMoELayer(nn.Module):
def __init__(
self,
config: AriaTextConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
prefix: str = "",
) -> None:
super().__init__()
@@ -415,7 +415,7 @@ class AriaProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(AriaProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_num_image_tokens(self) -> int:
@@ -436,7 +436,7 @@ class AriaDummyInputsBuilder(BaseDummyInputsBuilder[AriaProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
vision_config = self.info.get_vision_config()
@@ -517,7 +517,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|fim_prefix|><|img|><|fim_suffix|>"
@@ -562,7 +562,7 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[AriaImagePixelInputs]:
) -> AriaImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
pixel_mask = kwargs.pop("pixel_mask", None)
@@ -577,8 +577,8 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
def _create_patch_attention_mask(
self,
pixel_mask: Optional[torch.Tensor],
) -> Optional[torch.Tensor]:
pixel_mask: torch.Tensor | None,
) -> torch.Tensor | None:
if pixel_mask is None:
return None
@@ -628,10 +628,10 @@ class AriaForConditionalGeneration(nn.Module, SupportsMultiModal):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if inputs_embeds is None:
multimodal_embeddings = self.get_multimodal_embeddings(**kwargs)
inputs_embeds = self.get_input_embeddings(
+10 -10
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Adapted from https://github.com/huggingface/transformers/tree/main/src/transformers/models/aya_vision
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal
import torch
from torch import nn
@@ -139,7 +139,7 @@ class AyaVisionProcessingInfo(BaseProcessingInfo):
def get_image_processor(self, **kwargs: object) -> GotOcr2ImageProcessor:
return self.get_hf_processor(**kwargs).image_processor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_image_size_with_most_features(self) -> ImageSize:
@@ -187,7 +187,7 @@ class AyaVisionDummyInputsBuilder(BaseDummyInputsBuilder[AyaVisionProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_size = self.info.get_image_size_with_most_features()
@@ -331,7 +331,7 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
@@ -375,7 +375,7 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
self,
vision_tower: SiglipVisionModel,
pixel_values: torch.Tensor,
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
) -> torch.Tensor | tuple[torch.Tensor, ...]:
return vision_tower(
pixel_values.to(dtype=vision_tower.dtype),
feature_select_strategy=self.config.vision_feature_select_strategy,
@@ -395,7 +395,7 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[AyaVisionImagePixelInputs]:
) -> AyaVisionImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
num_patches = kwargs.pop("num_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -428,10 +428,10 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
@@ -446,5 +446,5 @@ class AyaVisionForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
+13 -14
View File
@@ -24,7 +24,6 @@
import math
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -98,7 +97,7 @@ class BaiChuanMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -130,8 +129,8 @@ class BaiChuanAttention(nn.Module):
position_embedding: str,
rope_theta: float = 10000,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -213,8 +212,8 @@ class BaiChuanDecoderLayer(nn.Module):
self,
config: PretrainedConfig,
position_embedding: str,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -246,7 +245,7 @@ class BaiChuanDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -305,9 +304,9 @@ class BaiChuanModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -422,9 +421,9 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -433,7 +432,7 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+16 -17
View File
@@ -26,7 +26,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
import torch.nn.functional as F
@@ -75,8 +74,8 @@ class BailingAttention(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
):
@@ -184,8 +183,8 @@ class BailingMLP(nn.Module):
self,
intermediate_size: int,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
reduce_results: Optional[bool] = True,
quant_config: QuantizationConfig | None = None,
reduce_results: bool | None = True,
prefix: str = "",
) -> None:
super().__init__()
@@ -218,8 +217,8 @@ class BailingMoE(nn.Module):
self,
intermediate_size: int,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
reduce_results: Optional[bool] = True,
quant_config: QuantizationConfig | None = None,
reduce_results: bool | None = True,
prefix: str = "",
):
super().__init__()
@@ -340,8 +339,8 @@ class BailingMoeBlock(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -370,7 +369,7 @@ class BailingMoeBlock(nn.Module):
self,
hidden_states: torch.Tensor,
position_ids: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
if residual is None:
residual = hidden_states
@@ -447,9 +446,9 @@ class BailingMoeModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -619,9 +618,9 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -630,7 +629,7 @@ class BailingMoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+14 -15
View File
@@ -4,7 +4,6 @@
# Added by the IBM Team, 2024
from collections.abc import Iterable
from typing import Optional
import torch
from torch import nn
@@ -52,7 +51,7 @@ class BambaMLP(nn.Module):
def __init__(
self,
config: BambaConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
) -> None:
super().__init__()
@@ -87,9 +86,9 @@ class BambaMixerDecoderLayer(nn.Module):
self,
config: BambaConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -119,7 +118,7 @@ class BambaMixerDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -141,9 +140,9 @@ class BambaAttentionDecoderLayer(nn.Module):
self,
config: BambaConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -235,7 +234,7 @@ class BambaAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -314,8 +313,8 @@ class BambaModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -497,8 +496,8 @@ class BambaForCausalLM(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
hidden_states = self.model(
@@ -510,7 +509,7 @@ class BambaForCausalLM(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+24 -25
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Set
from typing import Optional, Union
import torch
from torch import nn
@@ -66,7 +65,7 @@ class BertEmbedding(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
token_type_ids = _decode_token_type_ids(input_ids)
@@ -103,9 +102,9 @@ class BertPooler(Pooler):
def forward(
self,
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
hidden_states: torch.Tensor | list[torch.Tensor],
pooling_metadata: PoolingMetadata,
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
pooled_output = self.pooling(hidden_states, pooling_metadata)
if isinstance(pooled_output, list):
@@ -147,8 +146,8 @@ class BertLayer(nn.Module):
def __init__(
self,
config: BertConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -191,8 +190,8 @@ class BertAttention(nn.Module):
hidden_size: int,
num_attention_heads: int,
layer_norm_eps: float,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -225,8 +224,8 @@ class BertSelfAttention(nn.Module):
self,
hidden_size: int,
num_attention_heads: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -281,7 +280,7 @@ class BertSelfOutput(nn.Module):
self,
hidden_size: int,
layer_norm_eps: float,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -308,7 +307,7 @@ class BertIntermediate(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -333,7 +332,7 @@ class BertOutput(nn.Module):
hidden_size: int,
intermediate_size: int,
layer_norm_eps: float,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -383,8 +382,8 @@ class BertModel(nn.Module, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.embeddings(
input_ids=input_ids,
@@ -494,8 +493,8 @@ class BertEmbeddingModel(nn.Module, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
return self.model(
input_ids=input_ids,
@@ -636,11 +635,11 @@ class BertForSequenceClassification(nn.Module, SupportsCrossEncoding, SupportsQu
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
token_type_ids: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
token_type_ids: torch.Tensor | None = None,
) -> torch.Tensor:
if token_type_ids is not None:
assert self.bert.config.vocab_size < (1 << TOKEN_TYPE_SHIFT)
@@ -692,11 +691,11 @@ class BertForTokenClassification(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
token_type_ids: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
token_type_ids: torch.Tensor | None = None,
) -> torch.Tensor:
if token_type_ids is not None:
assert self.bert.config.vocab_size < (1 << TOKEN_TYPE_SHIFT)
+18 -19
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Optional
import torch
from torch import nn
@@ -67,7 +66,7 @@ class BertWithRopeEmbedding(nn.Module):
def forward(
self,
input_ids: torch.Tensor,
token_type_ids: Optional[torch.Tensor] = None,
token_type_ids: torch.Tensor | None = None,
) -> torch.Tensor:
input_shape = input_ids.size()
inputs_embeds = self.word_embeddings(input_ids)
@@ -91,10 +90,10 @@ class BertWithRopeAttention(nn.Module):
self,
hidden_size: int,
num_attention_heads: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
bias: bool = True,
rotary_kwargs: Optional[dict] = None,
rotary_kwargs: dict | None = None,
prefix: str = "",
):
super().__init__()
@@ -166,7 +165,7 @@ class BertWithRopeGatedMLP(nn.Module):
intermediate_size: int,
hidden_act: str,
bias: bool = True,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -200,7 +199,7 @@ class BertWithRopeMLP(nn.Module):
intermediate_size: int,
hidden_act: str,
bias: bool = True,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -235,8 +234,8 @@ class NomicMoE(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
params_dtype: Optional[torch.dtype] = None,
tp_size: Optional[int] = None,
params_dtype: torch.dtype | None = None,
tp_size: int | None = None,
):
super().__init__()
@@ -344,11 +343,11 @@ class BertWithRopeBlock(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
moe: bool = False,
bias: bool = True,
rotary_kwargs: Optional[dict] = None,
rotary_kwargs: dict | None = None,
prefix: str = "",
):
super().__init__()
@@ -406,7 +405,7 @@ class BertWithRopeEncoder(nn.Module):
self,
vllm_config: VllmConfig,
bias: bool = True,
rotary_kwargs: Optional[dict] = None,
rotary_kwargs: dict | None = None,
prefix: str = "",
):
super().__init__()
@@ -471,9 +470,9 @@ class BertWithRope(nn.Module, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
token_type_ids: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
token_type_ids: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -724,10 +723,10 @@ class GteNewForSequenceClassification(nn.Module, SupportsCrossEncoding):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
return self.new(
input_ids=input_ids,
+10 -11
View File
@@ -4,7 +4,6 @@
within a vision language model."""
from collections.abc import Iterable
from typing import Optional, Union
import torch
import torch.nn as nn
@@ -38,7 +37,7 @@ def get_blip_num_patches(*, image_size: int, patch_size: int) -> int:
# Adapted from https://github.com/huggingface/transformers/blob/v4.39.0/src/transformers/models/blip/modeling_blip.py#L164 # noqa
class BlipVisionEmbeddings(nn.Module):
def __init__(self, config: Union[BlipVisionConfig, Blip2VisionConfig]):
def __init__(self, config: BlipVisionConfig | Blip2VisionConfig):
super().__init__()
self.config = config
@@ -86,8 +85,8 @@ class BlipAttention(nn.Module):
def __init__(
self,
config: Union[BlipVisionConfig, Blip2VisionConfig],
quant_config: Optional[QuantizationConfig] = None,
config: BlipVisionConfig | Blip2VisionConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -151,7 +150,7 @@ class BlipMLP(nn.Module):
def __init__(
self,
config: BlipVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -186,7 +185,7 @@ class BlipEncoderLayer(nn.Module):
def __init__(
self,
config: BlipVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -228,8 +227,8 @@ class BlipEncoder(nn.Module):
def __init__(
self,
config: BlipVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
num_hidden_layers_override: Optional[int] = None,
quant_config: QuantizationConfig | None = None,
num_hidden_layers_override: int | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -268,10 +267,10 @@ class BlipVisionModel(nn.Module, SupportsQuant):
def __init__(
self,
config: BlipVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
require_post_norm: Optional[bool] = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> None:
super().__init__()
+21 -21
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal, TypeAlias
import torch
import torch.nn as nn
@@ -70,7 +70,7 @@ class Blip2ImageEmbeddingInputs(TensorSchema):
data: Annotated[torch.Tensor, TensorShape("bn", "f", "h")]
Blip2ImageInputs = Union[Blip2ImagePixelInputs, Blip2ImageEmbeddingInputs]
Blip2ImageInputs: TypeAlias = Blip2ImagePixelInputs | Blip2ImageEmbeddingInputs
class Blip2QFormerMultiHeadAttention(nn.Module):
@@ -78,8 +78,8 @@ class Blip2QFormerMultiHeadAttention(nn.Module):
self,
config: Blip2QFormerConfig,
*,
quant_config: Optional[QuantizationConfig],
cache_config: Optional[CacheConfig],
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
is_cross_attention: bool = False,
prefix: str = "",
) -> None:
@@ -123,7 +123,7 @@ class Blip2QFormerMultiHeadAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: Optional[torch.FloatTensor] = None,
encoder_hidden_states: torch.FloatTensor | None = None,
):
is_cross_attention = encoder_hidden_states is not None
@@ -179,8 +179,8 @@ class Blip2QFormerAttention(nn.Module):
self,
config: Blip2QFormerConfig,
*,
quant_config: Optional[QuantizationConfig],
cache_config: Optional[CacheConfig],
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
is_cross_attention: bool = False,
prefix: str = "",
) -> None:
@@ -199,7 +199,7 @@ class Blip2QFormerAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
encoder_hidden_states: Optional[torch.FloatTensor] = None,
encoder_hidden_states: torch.FloatTensor | None = None,
) -> tuple[torch.Tensor]:
self_output = self.attention(
hidden_states,
@@ -247,8 +247,8 @@ class Blip2QFormerLayer(nn.Module):
self,
config: Blip2QFormerConfig,
*,
quant_config: Optional[QuantizationConfig],
cache_config: Optional[CacheConfig],
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
layer_idx: int,
prefix: str = "",
) -> None:
@@ -340,8 +340,8 @@ class Blip2QFormerEncoder(nn.Module):
self,
config: Blip2QFormerConfig,
*,
quant_config: Optional[QuantizationConfig],
cache_config: Optional[CacheConfig],
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
prefix: str = "",
) -> None:
super().__init__()
@@ -385,8 +385,8 @@ class Blip2QFormerModel(nn.Module):
self,
config: Blip2QFormerConfig,
*,
quant_config: Optional[QuantizationConfig],
cache_config: Optional[CacheConfig],
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
prefix: str = "",
) -> None:
super().__init__()
@@ -426,7 +426,7 @@ class Blip2ProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(Blip2Config)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": 1}
def get_num_image_tokens(self) -> int:
@@ -442,7 +442,7 @@ class Blip2DummyInputsBuilder(BaseDummyInputsBuilder[Blip2ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
@@ -526,7 +526,7 @@ class Blip2ForConditionalGeneration(
merge_by_field_config = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return None
@@ -573,7 +573,7 @@ class Blip2ForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[Blip2ImageInputs]:
) -> Blip2ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -641,8 +641,8 @@ class Blip2ForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> IntermediateTensors:
"""Run forward pass for BLIP-2.
@@ -687,7 +687,7 @@ class Blip2ForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+12 -13
View File
@@ -22,7 +22,6 @@
import math
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -89,8 +88,8 @@ class BloomAttention(nn.Module):
def __init__(
self,
config: BloomConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -152,7 +151,7 @@ class BloomMLP(nn.Module):
def __init__(
self,
config: BloomConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
hidden_size = config.hidden_size
@@ -179,8 +178,8 @@ class BloomBlock(nn.Module):
def __init__(
self,
config: BloomConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -274,9 +273,9 @@ class BloomModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -356,9 +355,9 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -367,7 +366,7 @@ class BloomForCausalLM(nn.Module, SupportsPP, SupportsQuant):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+24 -24
View File
@@ -4,7 +4,7 @@
from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property
from itertools import islice
from typing import Annotated, Any, Literal, Optional, Union
from typing import Annotated, Any, Literal
import torch
import torch.nn as nn
@@ -94,7 +94,7 @@ class ChameleonProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(ChameleonProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": 1}
def get_num_image_tokens(self) -> int:
@@ -115,7 +115,7 @@ class ChameleonDummyInputsBuilder(BaseDummyInputsBuilder[ChameleonProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
config = self.info.get_hf_config()
@@ -225,7 +225,7 @@ class ChameleonMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
) -> None:
super().__init__()
@@ -262,11 +262,11 @@ class ChameleonAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 4096,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -357,8 +357,8 @@ class ChameleonDecoderLayer(nn.Module):
def __init__(
self,
config: ChameleonConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -403,8 +403,8 @@ class ChameleonDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
if residual is None:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
@@ -426,8 +426,8 @@ class ChameleonSwinDecoderLayer(nn.Module):
def __init__(
self,
config: ChameleonConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -472,7 +472,7 @@ class ChameleonSwinDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
residual = hidden_states
hidden_states = self.self_attn(
@@ -896,11 +896,11 @@ class ChameleonModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -941,7 +941,7 @@ class ChameleonForConditionalGeneration(
}
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
@@ -975,7 +975,7 @@ class ChameleonForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[ChameleonImagePixelInputs]:
) -> ChameleonImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
if pixel_values is None:
@@ -1008,10 +1008,10 @@ class ChameleonForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
@@ -1023,7 +1023,7 @@ class ChameleonForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
# Disallow image tokens which does not include special
+15 -16
View File
@@ -7,7 +7,6 @@
import json
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -50,8 +49,8 @@ class GLMAttention(nn.Module):
def __init__(
self,
config: ChatGLMConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -144,7 +143,7 @@ class GLMMLP(nn.Module):
def __init__(
self,
config: ChatGLMConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -190,8 +189,8 @@ class GLMBlock(nn.Module):
def __init__(
self,
config: ChatGLMConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -263,8 +262,8 @@ class GLMTransformer(nn.Module):
def __init__(
self,
config: ChatGLMConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -295,7 +294,7 @@ class GLMTransformer(nn.Module):
self,
hidden_states: torch.Tensor,
position_ids: torch.Tensor,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
for layer in islice(self.layers, self.start_layer, self.end_layer):
hidden_states = layer(
hidden_states=hidden_states, position_ids=position_ids
@@ -361,10 +360,10 @@ class ChatGLMModel(nn.Module, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -459,7 +458,7 @@ class ChatGLMBaseModel(nn.Module):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -491,9 +490,9 @@ class ChatGLMForCausalLM(ChatGLMBaseModel, SupportsLoRA, SupportsPP, SupportsQua
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
+44 -44
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, Sequence
from functools import cached_property
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal
import torch
import torch.nn as nn
@@ -125,7 +125,7 @@ class CLIPProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(CLIPProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": 1}
def get_num_image_tokens(
@@ -169,7 +169,7 @@ class CLIPDummyInputsBuilder(BaseDummyInputsBuilder[CLIPProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
@@ -199,12 +199,12 @@ class CLIPMultiModalProcessor(BaseMultiModalProcessor[CLIPProcessingInfo]):
def apply(
self,
prompt: Union[str, list[int]],
prompt: str | list[int],
mm_data: MultiModalDataDict,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Optional[Mapping[str, object]] = None,
tokenization_kwargs: Mapping[str, object] | None = None,
*,
mm_uuids: Optional[MultiModalUUIDDict] = None,
mm_uuids: MultiModalUUIDDict | None = None,
) -> MultiModalInputs:
if prompt and mm_data:
raise ValueError(
@@ -286,9 +286,9 @@ class CLIPTextEmbeddings(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
position_ids: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if inputs_embeds is None:
if input_ids is None:
@@ -350,11 +350,11 @@ class CLIPVisionEmbeddings(nn.Module):
class CLIPAttention(nn.Module):
def __init__(
self,
config: Union[CLIPTextConfig, CLIPVisionConfig],
quant_config: Optional[QuantizationConfig] = None,
config: CLIPTextConfig | CLIPVisionConfig,
quant_config: QuantizationConfig | None = None,
*,
prefix: str = "",
attn_cls: Union[type[Attention], type[MultiHeadAttention]],
attn_cls: type[Attention] | type[MultiHeadAttention],
) -> None:
super().__init__()
@@ -412,8 +412,8 @@ class CLIPAttention(nn.Module):
class CLIPMLP(nn.Module):
def __init__(
self,
config: Union[CLIPTextConfig, CLIPVisionConfig],
quant_config: Optional[QuantizationConfig] = None,
config: CLIPTextConfig | CLIPVisionConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -445,11 +445,11 @@ class CLIPMLP(nn.Module):
class CLIPEncoderLayer(nn.Module):
def __init__(
self,
config: Union[CLIPTextConfig, CLIPVisionConfig],
quant_config: Optional[QuantizationConfig] = None,
config: CLIPTextConfig | CLIPVisionConfig,
quant_config: QuantizationConfig | None = None,
*,
prefix: str = "",
attn_cls: Union[type[Attention], type[MultiHeadAttention]],
attn_cls: type[Attention] | type[MultiHeadAttention],
) -> None:
super().__init__()
self.self_attn = CLIPAttention(
@@ -488,12 +488,12 @@ class CLIPEncoder(nn.Module):
def __init__(
self,
config: Union[CLIPTextConfig, CLIPVisionConfig],
quant_config: Optional[QuantizationConfig] = None,
num_hidden_layers_override: Optional[int] = None,
config: CLIPTextConfig | CLIPVisionConfig,
quant_config: QuantizationConfig | None = None,
num_hidden_layers_override: int | None = None,
*,
prefix: str = "",
attn_cls: Union[type[Attention], type[MultiHeadAttention]],
attn_cls: type[Attention] | type[MultiHeadAttention],
) -> None:
super().__init__()
@@ -519,7 +519,7 @@ class CLIPEncoder(nn.Module):
self,
inputs_embeds: torch.Tensor,
return_all_hidden_states: bool,
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
hidden_states_pool = [inputs_embeds]
hidden_states = inputs_embeds
@@ -538,7 +538,7 @@ class CLIPTextTransformer(nn.Module):
def __init__(
self,
config: CLIPTextConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
prefix: str = "",
) -> None:
@@ -566,9 +566,9 @@ class CLIPTextTransformer(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
position_ids: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.embeddings(
input_ids=input_ids,
@@ -616,10 +616,10 @@ class CLIPVisionTransformer(nn.Module):
def __init__(
self,
config: CLIPVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
require_post_norm: Optional[bool] = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -669,8 +669,8 @@ class CLIPVisionTransformer(nn.Module):
self,
pixel_values: torch.Tensor,
*,
select_layers: Optional[list[int]] = None,
feature_select_strategy: Optional[VisionFeatureSelectStrategy] = None,
select_layers: list[int] | None = None,
feature_select_strategy: VisionFeatureSelectStrategy | None = None,
) -> torch.Tensor:
hidden_states = self.embeddings(pixel_values)
hidden_states = self.pre_layrnorm(hidden_states)
@@ -736,10 +736,10 @@ class CLIPVisionModel(nn.Module):
def __init__(
self,
config: CLIPVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
require_post_norm: Optional[bool] = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -755,8 +755,8 @@ class CLIPVisionModel(nn.Module):
def forward(
self,
pixel_values: torch.Tensor,
select_layers: Optional[list[int]] = None,
feature_select_strategy: Optional[VisionFeatureSelectStrategy] = None,
select_layers: list[int] | None = None,
feature_select_strategy: VisionFeatureSelectStrategy | None = None,
) -> torch.Tensor:
return self.vision_model(
pixel_values,
@@ -787,7 +787,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
merge_by_field_config = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return None
@@ -847,9 +847,9 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
def get_text_features(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
position_ids: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
pooled_output = self.text_model(
input_ids=input_ids,
@@ -864,7 +864,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
def get_image_features(
self,
pixel_values: torch.Tensor,
feature_select_strategy: Optional[VisionFeatureSelectStrategy] = None,
feature_select_strategy: VisionFeatureSelectStrategy | None = None,
) -> torch.Tensor:
if feature_select_strategy is None:
feature_select_strategy = _get_vision_feature_select_strategy(
@@ -883,7 +883,7 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[CLIPImagePixelInputs]:
) -> CLIPImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
if pixel_values is None:
return None
@@ -906,9 +906,9 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[torch.Tensor] = None,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
self._is_text_input = (
@@ -936,10 +936,10 @@ class CLIPEmbeddingModel(nn.Module, SupportsMultiModal, SupportsQuant):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor:
if intermediate_tensors is not None:
+9 -9
View File
@@ -4,7 +4,7 @@
"""Command-A-Vision (Cohere2Vision) multimodal model implementation for vLLM."""
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal
import torch
from torch import nn
@@ -148,7 +148,7 @@ class Cohere2VisionProcessingInfo(BaseProcessingInfo):
def get_image_processor(self, **kwargs: object):
return self.get_hf_processor(**kwargs).image_processor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_image_size_with_most_features(self) -> ImageSize:
@@ -163,7 +163,7 @@ class Cohere2VisionProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Cohere2VisionProcessor],
processor: Cohere2VisionProcessor | None,
) -> int:
"""
Calculate the number of image patches for a given image.
@@ -217,7 +217,7 @@ class Cohere2VisionDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
image_size = self.info.get_image_size_with_most_features()
@@ -404,7 +404,7 @@ class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[Cohere2VisionImagePixelInputs]:
) -> Cohere2VisionImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
num_patches = kwargs.pop("num_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -450,10 +450,10 @@ class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
@@ -468,5 +468,5 @@ class Cohere2VisionForConditionalGeneration(nn.Module, SupportsMultiModal, Suppo
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
+16 -17
View File
@@ -25,7 +25,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -94,8 +93,8 @@ class LayerNorm(nn.Module):
class CohereMLP(nn.Module):
def __init__(
self,
config: Union[CohereConfig, Cohere2Config],
quant_config: Optional[QuantizationConfig] = None,
config: CohereConfig | Cohere2Config,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -128,9 +127,9 @@ class CohereMLP(nn.Module):
class CohereAttention(nn.Module):
def __init__(
self,
config: Union[CohereConfig, Cohere2Config],
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
config: CohereConfig | Cohere2Config,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -241,9 +240,9 @@ class CohereAttention(nn.Module):
class CohereDecoderLayer(nn.Module):
def __init__(
self,
config: Union[CohereConfig, Cohere2Config],
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
config: CohereConfig | Cohere2Config,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -265,7 +264,7 @@ class CohereDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
residual = hidden_states
@@ -324,9 +323,9 @@ class CohereModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -452,9 +451,9 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -463,7 +462,7 @@ class CohereForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsQuant):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
is_not_lora = hasattr(self.model.embed_tokens, "weight")
if is_not_lora:
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
+18 -19
View File
@@ -3,7 +3,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
import torch.nn as nn
@@ -54,7 +53,7 @@ class DbrxRouter(nn.Module):
def __init__(
self,
config: DbrxConfig,
params_dtype: Optional[torch.dtype] = None,
params_dtype: torch.dtype | None = None,
):
super().__init__()
self.tp_size = get_tensor_model_parallel_world_size()
@@ -77,8 +76,8 @@ class DbrxExperts(FusedMoE):
def __init__(
self,
config: DbrxConfig,
quant_config: Optional[QuantizationConfig] = None,
params_dtype: Optional[torch.dtype] = None,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
prefix: str = "",
):
super().__init__(
@@ -157,8 +156,8 @@ class DbrxMoE(nn.Module):
def __init__(
self,
config: DbrxConfig,
quant_config: Optional[QuantizationConfig] = None,
params_dtype: Optional[torch.dtype] = None,
quant_config: QuantizationConfig | None = None,
params_dtype: torch.dtype | None = None,
prefix: str = "",
):
super().__init__()
@@ -189,8 +188,8 @@ class DbrxAttention(nn.Module):
def __init__(
self,
config: DbrxConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -270,8 +269,8 @@ class DbrxFusedNormAttention(nn.Module):
def __init__(
self,
config: DbrxConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -303,8 +302,8 @@ class DbrxBlock(nn.Module):
def __init__(
self,
config: DbrxConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -361,9 +360,9 @@ class DbrxModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -466,9 +465,9 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -477,7 +476,7 @@ class DbrxForCausalLM(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+16 -16
View File
@@ -26,7 +26,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -76,7 +76,7 @@ class DeepseekMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
@@ -108,7 +108,7 @@ class DeepseekMoE(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -203,10 +203,10 @@ class DeepseekAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -282,8 +282,8 @@ class DeepseekDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -328,7 +328,7 @@ class DeepseekDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
# Self Attention
if residual is None:
@@ -382,9 +382,9 @@ class DeepseekModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -489,9 +489,9 @@ class DeepseekForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -500,7 +500,7 @@ class DeepseekForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+2 -3
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -224,7 +223,7 @@ class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if inputs_embeds is not None:
raise NotImplementedError(
@@ -235,7 +234,7 @@ class EagleDeepseekV3ForCausalLM(DeepseekV3ForCausalLM):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+6 -7
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -30,7 +29,7 @@ class SharedHead(nn.Module):
self,
config: PretrainedConfig,
prefix: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -84,7 +83,7 @@ class DeepSeekMultiTokenPredictorLayer(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
@@ -136,7 +135,7 @@ class DeepSeekMultiTokenPredictor(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
@@ -180,8 +179,8 @@ class DeepSeekMTP(nn.Module, SupportsPP):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
@@ -193,7 +192,7 @@ class DeepSeekMTP(nn.Module, SupportsPP):
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+35 -35
View File
@@ -27,7 +27,7 @@
import typing
from collections.abc import Callable, Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -106,7 +106,7 @@ class DeepseekV2MLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
is_sequence_parallel=False,
prefix: str = "",
@@ -150,9 +150,9 @@ class DeepseekV2MLP(nn.Module):
class DeepseekV2MoE(nn.Module):
def __init__(
self,
config: Union[DeepseekV2Config, DeepseekV3Config],
config: DeepseekV2Config | DeepseekV3Config,
parallel_config: ParallelConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -301,7 +301,7 @@ class DeepseekV2Attention(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Union[DeepseekV2Config, DeepseekV3Config],
config: DeepseekV2Config | DeepseekV3Config,
hidden_size: int,
num_heads: int,
qk_nope_head_dim: int,
@@ -310,11 +310,11 @@ class DeepseekV2Attention(nn.Module):
q_lora_rank: int,
kv_lora_rank: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
topk_indices_buffer: Optional[torch.Tensor] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
topk_indices_buffer: torch.Tensor | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -554,12 +554,12 @@ def sparse_attn_indexer(
k: torch.Tensor,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: Optional[str],
scale_fmt: str | None,
topk_tokens: int,
head_dim: int,
max_model_len: int,
total_seq_lens: int,
topk_indices_buffer: Optional[torch.Tensor],
topk_indices_buffer: torch.Tensor | None,
) -> torch.Tensor:
# careful! this will be None in dummy run
attn_metadata = get_forward_context().attn_metadata
@@ -727,12 +727,12 @@ def sparse_attn_indexer_fake(
k: torch.Tensor,
weights: torch.Tensor,
quant_block_size: int,
scale_fmt: Optional[str],
scale_fmt: str | None,
topk_tokens: int,
head_dim: int,
max_model_len: int,
total_seq_lens: int,
topk_indices_buffer: Optional[torch.Tensor],
topk_indices_buffer: torch.Tensor | None,
) -> torch.Tensor:
# profile run
# NOTE(Chen): create the max possible flattened_kv. So that
@@ -758,12 +758,12 @@ class Indexer(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Union[DeepseekV2Config, DeepseekV3Config],
config: DeepseekV2Config | DeepseekV3Config,
hidden_size: int,
q_lora_rank: int,
quant_config: Optional[QuantizationConfig],
cache_config: Optional[CacheConfig],
topk_indices_buffer: Optional[torch.Tensor],
quant_config: QuantizationConfig | None,
cache_config: CacheConfig | None,
topk_indices_buffer: torch.Tensor | None,
prefix: str = "",
):
super().__init__()
@@ -880,21 +880,21 @@ class DeepseekV2MLAAttention(nn.Module):
def __init__(
self,
vllm_config: VllmConfig,
config: Union[DeepseekV2Config, DeepseekV3Config],
config: DeepseekV2Config | DeepseekV3Config,
hidden_size: int,
num_heads: int,
qk_nope_head_dim: int,
qk_rope_head_dim: int,
v_head_dim: int,
q_lora_rank: Optional[int],
q_lora_rank: int | None,
kv_lora_rank: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
topk_indices_buffer: Optional[torch.Tensor] = None,
topk_indices_buffer: torch.Tensor | None = None,
) -> None:
super().__init__()
self.hidden_size = hidden_size
@@ -1045,8 +1045,8 @@ class DeepseekV2DecoderLayer(nn.Module):
self,
vllm_config: VllmConfig,
prefix: str,
config: Optional[DeepseekV2Config] = None,
topk_indices_buffer: Optional[torch.Tensor] = None,
config: DeepseekV2Config | None = None,
topk_indices_buffer: torch.Tensor | None = None,
) -> None:
super().__init__()
@@ -1117,7 +1117,7 @@ class DeepseekV2DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
# Self Attention
if residual is None:
@@ -1212,9 +1212,9 @@ class DeepseekV2Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -1347,9 +1347,9 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts, SupportsLoR
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -1358,7 +1358,7 @@ class DeepseekV2ForCausalLM(nn.Module, SupportsPP, MixtureOfExperts, SupportsLoR
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -1497,8 +1497,8 @@ class DeepseekV3ForCausalLM(DeepseekV2ForCausalLM):
# Compatibility with
# https://huggingface.co/deepseek-ai/DeepSeek-V3-Base/blob/main/configuration_deepseek.py
def get_spec_layer_idx_from_weight_name(
config: Union[DeepseekV2Config, DeepseekV3Config], weight_name: str
) -> Optional[int]:
config: DeepseekV2Config | DeepseekV3Config, weight_name: str
) -> int | None:
if (
hasattr(config, "num_nextn_predict_layers")
and config.num_nextn_predict_layers > 0
+15 -17
View File
@@ -6,7 +6,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal, TypeAlias
import torch
import torch.nn as nn
@@ -88,14 +88,12 @@ class DeepseekVL2VImageEmbeddingInputs(TensorSchema):
"""
type: Literal["image_embeds"]
data: Annotated[
Union[torch.Tensor, list[torch.Tensor]], TensorShape("bn", "f", "h")
]
data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("bn", "f", "h")]
DeepseekVL2ImageInputs = Union[
DeepseekVL2ImagePixelInputs, DeepseekVL2VImageEmbeddingInputs
]
DeepseekVL2ImageInputs: TypeAlias = (
DeepseekVL2ImagePixelInputs | DeepseekVL2VImageEmbeddingInputs
)
class MlpProjector(nn.Module):
@@ -161,7 +159,7 @@ class DeepseekVL2ProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(DeepseekVLV2Processor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_num_image_tokens(
@@ -214,7 +212,7 @@ class DeepseekVL2DummyInputsBuilder(BaseDummyInputsBuilder[DeepseekVL2Processing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
@@ -310,11 +308,11 @@ class DeepseekVL2MultiModalProcessor(
def _cached_apply_hf_processor(
self,
prompt: Union[str, list[int]],
prompt: str | list[int],
mm_data_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
mm_uuids: Optional[MultiModalUUIDDict] = None,
mm_uuids: MultiModalUUIDDict | None = None,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
# The processor logic is different for len(images) <= 2 vs > 2
# Since the processing cache assumes that the processor output is
@@ -353,7 +351,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
@@ -454,7 +452,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def _init_vision_module(
self,
vision_config: VisionEncoderConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
prefix: str = "",
) -> nn.Module:
# TODO: refactor vision model through timm wrapper from transformers
@@ -480,7 +478,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[DeepseekVL2ImageInputs]:
) -> DeepseekVL2ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
images_spatial_crop = kwargs.pop("images_spatial_crop", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -637,8 +635,8 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
):
if intermediate_tensors is not None:
@@ -653,7 +651,7 @@ class DeepseekVLV2ForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+16 -16
View File
@@ -27,7 +27,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -80,7 +80,7 @@ class Dots1MLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
@@ -117,7 +117,7 @@ class Dots1MoE(nn.Module):
def __init__(
self,
config: Dots1Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -203,10 +203,10 @@ class Dots1Attention(nn.Module):
num_kv_heads: int,
config: Dots1Config,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -289,8 +289,8 @@ class Dots1DecoderLayer(nn.Module):
config: Dots1Config,
prefix: str,
model_config: ModelConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.hidden_size = config.hidden_size
@@ -338,7 +338,7 @@ class Dots1DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
if residual is None:
residual = hidden_states
@@ -403,9 +403,9 @@ class Dots1Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -546,9 +546,9 @@ class Dots1ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids,
positions,
@@ -560,7 +560,7 @@ class Dots1ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+22 -22
View File
@@ -1,7 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal, TypeAlias
import torch
import torch.nn as nn
@@ -92,7 +92,7 @@ class DotsOCRImageEmbeddingInputs(TensorSchema):
image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
DotsOCRImageInputs = Union[DotsOCRImagePixelInputs, DotsOCRImageEmbeddingInputs]
DotsOCRImageInputs: TypeAlias = DotsOCRImagePixelInputs | DotsOCRImageEmbeddingInputs
class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
@@ -104,7 +104,7 @@ class DotsOCRDummyInputsBuilder(Qwen2VLDummyInputsBuilder):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
@@ -134,7 +134,7 @@ class DotsOCRProcessingInfo(Qwen2VLProcessingInfo):
return config
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_mm_max_tokens_per_item(
@@ -253,7 +253,7 @@ class DotsVisionAttention(nn.Module):
num_heads: int = 16,
bias: bool = True,
*,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -316,10 +316,10 @@ class DotsVisionAttention(nn.Module):
self,
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb: Optional[torch.Tensor] = None,
rotary_pos_emb: torch.Tensor | None = None,
*,
max_seqlen: Optional[int] = None,
seqlens: Optional[list[int]] = None,
max_seqlen: int | None = None,
seqlens: list[int] | None = None,
) -> torch.Tensor:
# [S, C] -> [S, B=1, C]
x = hidden_states.unsqueeze(1)
@@ -394,7 +394,7 @@ class DotsSwiGLUFFN(nn.Module):
self,
config,
*,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
):
@@ -507,7 +507,7 @@ class DotsVisionBlock(nn.Module):
self,
config,
*,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
):
@@ -537,8 +537,8 @@ class DotsVisionBlock(nn.Module):
*,
cu_seqlens: torch.Tensor,
rotary_pos_emb: torch.Tensor,
max_seqlen: Optional[int] = None,
seqlens: Optional[list[int]] = None,
max_seqlen: int | None = None,
seqlens: list[int] | None = None,
) -> torch.Tensor:
hidden_states = hidden_states + self.attn(
self.norm1(hidden_states),
@@ -555,10 +555,10 @@ class DotsVisionTransformer(nn.Module):
def __init__(
self,
config: DotsVisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
require_post_norm: Optional[bool] = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -653,7 +653,7 @@ class DotsVisionTransformer(nn.Module):
def compute_attn_mask_seqlen(
self, cu_seqlens: torch.Tensor
) -> tuple[Optional[int], Optional[list[int]]]:
) -> tuple[int | None, list[int] | None]:
max_seqlen, seqlens = None, None
if (
self.attn_backend == _Backend.FLASH_ATTN
@@ -734,7 +734,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
supports_encoder_tp_data = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|img|><|imgpad|><|endofimg|>"
@@ -765,7 +765,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[DotsOCRImageInputs]:
) -> DotsOCRImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
@@ -834,10 +834,10 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
elif inputs_embeds is None:
@@ -861,7 +861,7 @@ class DotsOCRForCausalLM(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+17 -17
View File
@@ -25,7 +25,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -83,7 +83,7 @@ class Ernie4_5_MoeMLP(nn.Module):
intermediate_size: int,
hidden_act: str,
use_bias: bool = False,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
@@ -120,7 +120,7 @@ class Ernie4_5_MoeMoE(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
):
@@ -229,14 +229,14 @@ class Ernie4_5_MoeAttention(nn.Module):
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: Optional[int] = None,
head_dim: int | None = None,
rope_theta: float = 500000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 131072,
rms_norm_eps: float = 1e-05,
qkv_bias: bool = False,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -323,8 +323,8 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
) -> None:
@@ -391,7 +391,7 @@ class Ernie4_5_MoeDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
# Self Attention
if residual is None:
@@ -467,9 +467,9 @@ class Ernie4_5_MoeModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -737,9 +737,9 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -748,7 +748,7 @@ class Ernie4_5_MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA, MixtureOfExpe
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+34 -34
View File
@@ -25,9 +25,9 @@
import itertools
import math
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Callable, Literal, Optional, Union
from typing import Annotated, Any, Literal
import numpy as np
import torch
@@ -162,7 +162,7 @@ class Ernie4_5_VisionAttention(nn.Module):
embed_dim: int,
num_heads: int,
projection_size: int,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -254,8 +254,8 @@ class Ernie4_5_VisionAttention(nn.Module):
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb: torch.Tensor,
max_seqlen: Optional[int] = None, # Only used for Flash Attention
seqlens: Optional[list[int]] = None, # Only used for xFormers
max_seqlen: int | None = None, # Only used for Flash Attention
seqlens: list[int] | None = None, # Only used for xFormers
) -> torch.Tensor:
# [s, b, c] --> [s, b, head * 3 * head_dim]
x, _ = self.qkv(x)
@@ -332,7 +332,7 @@ class Ernie4_5_VisionMLP(nn.Module):
in_features: int,
hidden_features: int,
act_layer: type[nn.Module] = QuickGELU,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -364,8 +364,8 @@ class Ernie4_5_VisionBlock(nn.Module):
num_heads: int,
mlp_ratio: float,
act_layer: type[nn.Module] = QuickGELU,
norm_layer: Optional[Callable[[int], nn.Module]] = None,
quant_config: Optional[QuantizationConfig] = None,
norm_layer: Callable[[int], nn.Module] | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -397,8 +397,8 @@ class Ernie4_5_VisionBlock(nn.Module):
hidden_states: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb: torch.Tensor,
max_seqlen: Optional[int] = None, # Only used for Flash Attention
seqlens: Optional[list[int]] = None, # Only used for xFormers
max_seqlen: int | None = None, # Only used for Flash Attention
seqlens: list[int] | None = None, # Only used for xFormers
) -> torch.Tensor:
hidden_states = hidden_states + self.attn(
self.norm1(hidden_states),
@@ -456,7 +456,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
self,
vision_config,
norm_eps: float = 1e-6,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -553,7 +553,7 @@ class Ernie4_5_VisionTransformer(nn.Module):
def compute_attn_mask_seqlen(
self, cu_seqlens: torch.Tensor
) -> tuple[Optional[int], Optional[list[int]]]:
) -> tuple[int | None, list[int] | None]:
max_seqlen, seqlens = None, None
if (
self.attn_backend == _Backend.FLASH_ATTN
@@ -659,15 +659,15 @@ Ernie4_5_VLVideoInputs = Ernie4_5_VLVideoPixelInputs
# === Vision Processor === #
def round_by_factor(number: Union[int, float], factor: int) -> int:
def round_by_factor(number: int | float, factor: int) -> int:
return round(number / factor) * factor
def ceil_by_factor(number: Union[int, float], factor: int) -> int:
def ceil_by_factor(number: int | float, factor: int) -> int:
return math.ceil(number / factor) * factor
def floor_by_factor(number: Union[int, float], factor: int) -> int:
def floor_by_factor(number: int | float, factor: int) -> int:
return math.floor(number / factor) * factor
@@ -901,7 +901,7 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
def get_image_processor(self, **kwargs: object):
return self.get_hf_processor(**kwargs).image_processor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
def get_mm_max_tokens_per_item(
@@ -920,7 +920,7 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
image_height: int,
num_frames: int = 1,
do_resize: bool = True,
image_processor: Optional[Any],
image_processor: Any | None,
) -> tuple[ImageSize, int]:
if image_processor is None:
image_processor = self.get_image_processor()
@@ -957,7 +957,7 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
image_processor: Optional[Any],
image_processor: Any | None,
) -> int:
_, num_image_tokens = self._get_vision_info(
image_width=image_width,
@@ -972,7 +972,7 @@ class Ernie4_5_VLProcessingInfo(BaseProcessingInfo):
image_width: int,
image_height: int,
num_frames: int,
image_processor: Optional[Any],
image_processor: Any | None,
) -> int:
_, num_video_tokens = self._get_vision_info(
image_width=image_width,
@@ -1237,7 +1237,7 @@ class Ernie4_5_VLDummyInputsBuilder(BaseDummyInputsBuilder[Ernie4_5_VLProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -1310,7 +1310,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|IMAGE_START|><|image@placeholder|><|IMAGE_END|>"
if modality.startswith("video"):
@@ -1356,7 +1356,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
"""compute logits"""
return self.language_model.compute_logits(hidden_states)
@@ -1395,12 +1395,12 @@ class Ernie4_5_VLMoeForConditionalGeneration(
cls,
input_tokens: list[int],
hf_config: PretrainedConfig,
image_grid_thw: Union[list[list[int]], torch.Tensor],
video_grid_thw: Union[list[list[int]], torch.Tensor],
image_grid_thw: list[list[int]] | torch.Tensor,
video_grid_thw: list[list[int]] | torch.Tensor,
context_len: int = 0,
seq_len: Optional[int] = None,
second_per_grid_ts: Optional[list[float]] = None,
audio_feature_lengths: Optional[torch.Tensor] = None,
seq_len: int | None = None,
second_per_grid_ts: list[float] | None = None,
audio_feature_lengths: torch.Tensor | None = None,
use_audio_in_video: bool = False,
) -> tuple[torch.Tensor, int]:
"""Get mrope input positions and delta value for Ernie VL."""
@@ -1540,7 +1540,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[Ernie4_5_VLImageInputs]:
) -> Ernie4_5_VLImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
@@ -1556,7 +1556,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[Ernie4_5_VLVideoInputs]:
) -> Ernie4_5_VLVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
@@ -1631,7 +1631,7 @@ class Ernie4_5_VLMoeForConditionalGeneration(
def get_multimodal_embeddings(
self, **kwargs: object
) -> Optional[MultiModalEmbeddings]:
) -> MultiModalEmbeddings | None:
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
if not modalities:
return None
@@ -1657,9 +1657,9 @@ class Ernie4_5_VLMoeForConditionalGeneration(
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[torch.Tensor] = None,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
@@ -1680,8 +1680,8 @@ class Ernie4_5_VLMoeForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
forward_kwargs = {
+19 -19
View File
@@ -25,7 +25,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -74,7 +74,7 @@ logger = init_logger(__name__)
class Ernie4_5_VLMoeMLP(Ernie4_5_MoeMLP):
def __init__(self, shared_experts: Optional[torch.nn.Module] = None, **kwargs):
def __init__(self, shared_experts: torch.nn.Module | None = None, **kwargs):
super().__init__(**kwargs)
self.shared_experts = shared_experts
@@ -91,15 +91,15 @@ class Ernie4_5_VLMoeAttention(nn.Module):
hidden_size: int,
num_heads: int,
num_kv_heads: int,
head_dim: Optional[int] = None,
head_dim: int | None = None,
rope_theta: float = 500000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
freq_allocation: int = 20,
max_position_embeddings: int = 131072,
rms_norm_eps: float = 1e-05,
qkv_bias: bool = False,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -192,7 +192,7 @@ class Ernie4_5_VLMoeMoE(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -390,8 +390,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -463,8 +463,8 @@ class Ernie4_5_VLMoeDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
visual_token_mask: Optional[torch.Tensor],
residual: torch.Tensor | None,
visual_token_mask: torch.Tensor | None,
**kwargs: object,
) -> torch.Tensor:
# Self Attention
@@ -551,11 +551,11 @@ class Ernie4_5_VLMoeModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
visual_token_mask: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
visual_token_mask: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -632,10 +632,10 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
)
@@ -644,7 +644,7 @@ class Ernie4_5_VLMoeForCausalLM(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+4 -5
View File
@@ -24,7 +24,6 @@
"""Inference-only Ernie-MTP model."""
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -121,7 +120,7 @@ class ErnieMultiTokenPredictor(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
@@ -169,8 +168,8 @@ class ErnieMTP(nn.Module, SupportsPP):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
assert spec_step_idx == 0, "ernie_mtp only support predict one token"
@@ -183,7 +182,7 @@ class ErnieMTP(nn.Module, SupportsPP):
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+19 -19
View File
@@ -27,7 +27,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -75,7 +75,7 @@ class ExaoneGatedMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
) -> None:
@@ -115,11 +115,11 @@ class ExaoneAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -209,11 +209,11 @@ class ExaoneBlockAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -246,8 +246,8 @@ class ExaoneDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -296,7 +296,7 @@ class ExaoneDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -369,11 +369,11 @@ class ExaoneModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -536,9 +536,9 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -547,7 +547,7 @@ class ExaoneForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+16 -16
View File
@@ -23,7 +23,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -72,7 +72,7 @@ class Exaone4GatedMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
) -> None:
@@ -112,11 +112,11 @@ class Exaone4Attention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 1000000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -222,8 +222,8 @@ class Exaone4DecoderLayer(nn.Module):
def __init__(
self,
config: Exaone4Config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -277,7 +277,7 @@ class Exaone4DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
residual = hidden_states
@@ -356,11 +356,11 @@ class Exaone4Model(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -523,9 +523,9 @@ class Exaone4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -534,7 +534,7 @@ class Exaone4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+13 -13
View File
@@ -23,7 +23,7 @@
import math
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
from typing import TypeAlias
import torch
from torch import nn
@@ -65,7 +65,7 @@ from .utils import (
maybe_prefix,
)
FalconConfig = Union[HF_FalconConfig, RWConfig]
FalconConfig: TypeAlias = HF_FalconConfig | RWConfig
def _get_alibi_slopes(total_num_heads: int) -> torch.Tensor:
@@ -95,8 +95,8 @@ class FalconAttention(nn.Module):
def __init__(
self,
config: FalconConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -226,7 +226,7 @@ class FalconMLP(nn.Module):
def __init__(
self,
config: FalconConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
hidden_size = config.hidden_size
@@ -265,8 +265,8 @@ class FalconDecoderLayer(nn.Module):
def __init__(
self,
config: FalconConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -401,9 +401,9 @@ class FalconModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -517,8 +517,8 @@ class FalconForCausalLM(nn.Module, SupportsPP):
self,
input_ids: torch.LongTensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
@@ -528,7 +528,7 @@ class FalconForCausalLM(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+16 -17
View File
@@ -4,7 +4,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional
import torch
from torch import nn
@@ -52,7 +51,7 @@ class FalconH1MLP(nn.Module):
def __init__(
self,
config: FalconH1Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
) -> None:
super().__init__()
@@ -91,9 +90,9 @@ class FalconH1SSMDecoderLayer(nn.Module):
def __init__(
self,
config: FalconH1Config,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -188,7 +187,7 @@ class FalconH1SSMDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
output = torch.empty_like(hidden_states)
@@ -204,8 +203,8 @@ class FalconH1AttentionDecoderLayer(nn.Module):
def __init__(
self,
config: FalconH1Config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -301,7 +300,7 @@ class FalconH1AttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
hidden_states = self.self_attention(
@@ -326,9 +325,9 @@ class FalconH1ParallelHybrid(nn.Module):
self,
config: FalconH1Config,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -467,8 +466,8 @@ class FalconH1Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -610,8 +609,8 @@ class FalconH1ForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP, Is
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
hidden_states = self.model(
@@ -626,7 +625,7 @@ class FalconH1ForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP, Is
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+2 -4
View File
@@ -14,8 +14,6 @@
# limitations under the License.
"""Inference-only FlexOlmo model compatible with HuggingFace weights."""
from typing import Optional
import torch
from torch import nn
@@ -128,8 +126,8 @@ class FlexOlmoDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
# Attention block.
residual = hidden_states
hidden_states = self.self_attn(positions, hidden_states)
+8 -8
View File
@@ -20,7 +20,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional
from typing import Annotated, Literal
import torch
import torch.nn as nn
@@ -87,7 +87,7 @@ class FuyuProcessingInfo(BaseProcessingInfo):
def get_image_processor(self, **kwargs: object) -> FuyuImageProcessor:
return self.get_hf_processor(**kwargs).image_processor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": 1}
def get_image_feature_grid_size(
@@ -142,7 +142,7 @@ class FuyuDummyInputsBuilder(BaseDummyInputsBuilder[FuyuProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
@@ -271,7 +271,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return None
@@ -305,7 +305,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[FuyuImagePatchInputs]:
) -> FuyuImagePatchInputs | None:
image_patches = kwargs.pop("image_patches", None)
patches_per_image = kwargs.pop("patches_per_image", None)
@@ -344,8 +344,8 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
):
if intermediate_tensors is not None:
@@ -362,7 +362,7 @@ class FuyuForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.language_model.logits_processor(
self.language_model.lm_head, hidden_states
)
+17 -18
View File
@@ -20,7 +20,6 @@
from collections.abc import Iterable
from functools import cache
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -59,8 +58,8 @@ logger = init_logger(__name__)
@cache
def _get_gemma_act_fn(
hidden_act: Optional[str],
hidden_activation: Optional[str],
hidden_act: str | None,
hidden_activation: str | None,
) -> nn.Module:
if hidden_activation is None:
if hidden_act is not None:
@@ -92,9 +91,9 @@ class GemmaMLP(nn.Module):
self,
hidden_size: int,
intermediate_size: int,
hidden_act: Optional[str] = None,
hidden_activation: Optional[str] = None,
quant_config: Optional[QuantizationConfig] = None,
hidden_act: str | None = None,
hidden_activation: str | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -130,8 +129,8 @@ class GemmaAttention(nn.Module):
head_dim: int,
max_position_embeddings: int = 8192,
rope_theta: float = 10000,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -207,8 +206,8 @@ class GemmaDecoderLayer(nn.Module):
def __init__(
self,
config: GemmaConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -241,7 +240,7 @@ class GemmaDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -301,9 +300,9 @@ class GemmaModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -406,9 +405,9 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -417,7 +416,7 @@ class GemmaForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
return logits
+15 -16
View File
@@ -18,7 +18,6 @@
# limitations under the License.
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -66,7 +65,7 @@ class Gemma2MLP(nn.Module):
intermediate_size: int,
hidden_act: str,
hidden_activation: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -100,9 +99,9 @@ class Gemma2Attention(nn.Module):
head_dim: int,
max_position_embeddings: int,
rope_theta: float,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
attn_logits_soft_cap: Optional[float] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
attn_logits_soft_cap: float | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -183,8 +182,8 @@ class Gemma2DecoderLayer(nn.Module):
def __init__(
self,
config: Gemma2Config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -225,7 +224,7 @@ class Gemma2DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
@@ -284,11 +283,11 @@ class Gemma2Model(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -406,9 +405,9 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -417,7 +416,7 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
return logits
+15 -16
View File
@@ -17,7 +17,6 @@
# limitations under the License.
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
import torch.nn.functional as F
@@ -66,7 +65,7 @@ class Gemma3MLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_activation: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -108,9 +107,9 @@ class Gemma3Attention(nn.Module):
num_kv_heads: int,
head_dim: int,
max_position_embeddings: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
attn_logits_soft_cap: Optional[float] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
attn_logits_soft_cap: float | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -295,8 +294,8 @@ class Gemma3DecoderLayer(nn.Module):
def __init__(
self,
config: Gemma3TextConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -336,7 +335,7 @@ class Gemma3DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
@@ -401,12 +400,12 @@ class Gemma3Model(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -549,10 +548,10 @@ class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds, **kwargs
)
@@ -561,7 +560,7 @@ class Gemma3ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
return logits
+11 -11
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, Optional
from typing import Annotated, Any, Literal
import torch
from torch import nn
@@ -82,7 +82,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(Gemma3Processor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def _resolve_image_kwargs(
@@ -112,7 +112,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Gemma3Processor],
processor: Gemma3Processor | None,
) -> int:
if processor is None:
processor = self.get_hf_processor()
@@ -182,7 +182,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Gemma3Processor],
processor: Gemma3Processor | None,
) -> PromptUpdateDetails[str]:
if processor is None:
processor = self.get_hf_processor()
@@ -217,7 +217,7 @@ class Gemma3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Gemma3Processor],
processor: Gemma3Processor | None,
) -> int:
if processor is None:
processor = self.get_hf_processor()
@@ -256,7 +256,7 @@ class Gemma3DummyInputsBuilder(BaseDummyInputsBuilder[Gemma3ProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
@@ -510,7 +510,7 @@ class Gemma3ForConditionalGeneration(
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<start_of_image>"
@@ -555,7 +555,7 @@ class Gemma3ForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[Gemma3ImageInputs]:
) -> Gemma3ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
num_patches = kwargs.pop("num_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -609,8 +609,8 @@ class Gemma3ForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> IntermediateTensors:
if intermediate_tensors is not None:
@@ -692,7 +692,7 @@ class Gemma3ForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+23 -24
View File
@@ -16,7 +16,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch import nn
@@ -196,7 +195,7 @@ class Gemma3nLaurelBlock(nn.Module):
laurel_rank: int,
rms_norm_eps: float,
*,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str,
) -> None:
super().__init__()
@@ -236,7 +235,7 @@ class Gemma3nMLP(nn.Module):
intermediate_size: int,
hidden_activation: str,
activation_sparsity: float = 0.0,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -285,8 +284,8 @@ class Gemma3nAttention(nn.Module):
num_kv_heads: int,
head_dim: int,
max_position_embeddings: int,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -412,8 +411,8 @@ class Gemma3nDecoderLayer(nn.Module):
def __init__(
self,
config: Gemma3nTextConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -650,7 +649,7 @@ class Gemma3nSelfDecoder(nn.Module):
def get_per_layer_inputs(
self,
hidden_states_0: torch.Tensor,
per_layer_inputs: Optional[torch.Tensor],
per_layer_inputs: torch.Tensor | None,
) -> torch.Tensor:
per_layer_projection = self.per_layer_model_projection(hidden_states_0)
per_layer_projection = per_layer_projection.reshape(
@@ -687,8 +686,8 @@ class Gemma3nSelfDecoder(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
per_layer_inputs: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
per_layer_inputs: torch.Tensor | None = None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
if inputs_embeds is not None:
@@ -870,8 +869,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
per_layer_inputs: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
per_layer_inputs: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
logits_indices_padded, num_logits_indices = None, None
@@ -947,8 +946,8 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
per_layer_inputs: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
per_layer_inputs: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
hidden_states, per_layer_inputs = self.self_decoder(
@@ -990,13 +989,13 @@ class Gemma3nTextModel(nn.Module, SupportsQuant):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
per_layer_inputs: Optional[torch.Tensor] = None,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
per_layer_inputs: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if self.fast_prefill_enabled:
hidden_states = self.fast_prefill_forward(
input_ids,
@@ -1116,11 +1115,11 @@ class Gemma3nForCausalLM(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
*,
per_layer_inputs: Optional[torch.Tensor] = None,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
per_layer_inputs: torch.Tensor | None = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids,
positions,
@@ -1134,7 +1133,7 @@ class Gemma3nForCausalLM(nn.Module):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.model.embed_tokens, hidden_states)
return logits
+10 -11
View File
@@ -24,7 +24,6 @@
"""Inference-only GLM-4-0414 model compatible with HuggingFace weights."""
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch import nn
@@ -56,12 +55,12 @@ class Glm4Attention(nn.Module):
num_heads: int,
num_kv_heads: int,
max_position: int = 4096 * 32,
head_dim: Optional[int] = None,
head_dim: int | None = None,
qkv_bias: bool = False,
rope_theta: float = 10000,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
rope_scaling: Optional[tuple] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
rope_scaling: tuple | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
@@ -142,7 +141,7 @@ class Glm4DecoderLayer(nn.Module):
self,
vllm_config: VllmConfig,
prefix: str = "",
config: Optional[Glm4Config] = None,
config: Glm4Config | None = None,
) -> None:
super().__init__()
@@ -189,7 +188,7 @@ class Glm4DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -285,9 +284,9 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -296,7 +295,7 @@ class Glm4ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+26 -26
View File
@@ -27,9 +27,9 @@
"""Inference-only GLM-4V model compatible with HuggingFace weights."""
import math
from collections.abc import Iterable, Mapping, Sequence
from collections.abc import Callable, Iterable, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Callable, Literal, Optional, Union
from typing import Annotated, Any, Literal, TypeAlias
import numpy as np
import torch
@@ -140,7 +140,7 @@ class Glm4vImageEmbeddingInputs(TensorSchema):
image_grid_thw: Annotated[torch.Tensor, TensorShape("n", 3)]
Glm4vImageInputs = Union[Glm4vImagePixelInputs, Glm4vImageEmbeddingInputs]
Glm4vImageInputs: TypeAlias = Glm4vImagePixelInputs | Glm4vImageEmbeddingInputs
class Glm4vVideoPixelInputs(TensorSchema):
@@ -176,7 +176,7 @@ class Glm4vVideoEmbeddingInputs(TensorSchema):
video_grid_thw: Annotated[torch.Tensor, TensorShape("f", 3)]
Glm4vVideoInputs = Union[Glm4vVideoPixelInputs, Glm4vVideoEmbeddingInputs]
Glm4vVideoInputs: TypeAlias = Glm4vVideoPixelInputs | Glm4vVideoEmbeddingInputs
# ==== Vision Encoder ==== #
@@ -187,7 +187,7 @@ class Glm4vVisionMLP(nn.Module):
in_features: int,
hidden_features: int,
bias: bool = False,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
):
@@ -244,7 +244,7 @@ class Glm4vVisionAttention(nn.Module):
embed_dim: int,
num_heads: int,
projection_size: int,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -334,8 +334,8 @@ class Glm4vVisionAttention(nn.Module):
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb: torch.Tensor,
max_seqlen: Optional[int] = None, # Only used for Flash Attention
seqlens: Optional[list[int]] = None, # Only used for xFormers
max_seqlen: int | None = None, # Only used for Flash Attention
seqlens: list[int] | None = None, # Only used for xFormers
) -> torch.Tensor:
# [s, b, c] --> [s, b, head * 3 * head_dim]
x, _ = self.qkv(x)
@@ -413,8 +413,8 @@ class Glm4vVisionBlock(nn.Module):
dim: int,
num_heads: int,
mlp_hidden_dim: int,
norm_layer: Optional[Callable[[int], nn.Module]] = None,
quant_config: Optional[QuantizationConfig] = None,
norm_layer: Callable[[int], nn.Module] | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -445,8 +445,8 @@ class Glm4vVisionBlock(nn.Module):
x: torch.Tensor,
cu_seqlens: torch.Tensor,
rotary_pos_emb: torch.Tensor,
max_seqlen: Optional[int] = None, # Only used for Flash Attention
seqlens: Optional[list[int]] = None, # Only used for xFormers
max_seqlen: int | None = None, # Only used for Flash Attention
seqlens: list[int] | None = None, # Only used for xFormers
) -> torch.Tensor:
x_attn = self.attn(
self.norm1(x),
@@ -495,7 +495,7 @@ class Glm4vPatchMerger(nn.Module):
self,
d_model: int,
context_dim: int,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
use_data_parallel: bool = False,
@@ -693,7 +693,7 @@ class Glm4vVisionTransformer(nn.Module):
self,
vision_config: Glm4vVisionConfig,
norm_eps: float = 1e-6,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -809,7 +809,7 @@ class Glm4vVisionTransformer(nn.Module):
def compute_attn_mask_seqlen(
self,
cu_seqlens: torch.Tensor,
) -> tuple[Optional[int], Optional[list[int]]]:
) -> tuple[int | None, list[int] | None]:
max_seqlen, seqlens = None, None
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
if (
@@ -904,7 +904,7 @@ class Glm4vProcessingInfo(BaseProcessingInfo):
def get_tokenizer(self):
return self.ctx.tokenizer
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": 1}
def get_image_processor(self, **kwargs: object) -> Glm4vImageProcessor:
@@ -1141,7 +1141,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -1177,7 +1177,7 @@ class Glm4vDummyInputsBuilder(BaseDummyInputsBuilder[Glm4vProcessingInfo]):
height: int,
num_frames: int,
num_videos: int,
overrides: Optional[VideoDummyOptions] = None,
overrides: VideoDummyOptions | None = None,
) -> list[VideoItem]:
if overrides:
if overrides.num_frames:
@@ -1419,7 +1419,7 @@ class Glm4vForConditionalGeneration(
supports_encoder_tp_data = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|begin_of_image|><|image|><|end_of_image|>"
if modality.startswith("video"):
@@ -1465,7 +1465,7 @@ class Glm4vForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[Glm4vImageInputs]:
) -> Glm4vImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
@@ -1489,7 +1489,7 @@ class Glm4vForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[Glm4vVideoInputs]:
) -> Glm4vVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
@@ -1594,7 +1594,7 @@ class Glm4vForConditionalGeneration(
def get_multimodal_embeddings(
self, **kwargs: object
) -> Optional[MultiModalEmbeddings]:
) -> MultiModalEmbeddings | None:
mm_input_by_modality = self._parse_and_validate_multimodal_inputs(**kwargs)
if not mm_input_by_modality:
return None
@@ -1619,10 +1619,10 @@ class Glm4vForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for GLM-4V.
Args:
@@ -1652,7 +1652,7 @@ class Glm4vForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+18 -18
View File
@@ -26,7 +26,7 @@
import typing
from collections.abc import Callable, Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -81,7 +81,7 @@ class Glm4MoeMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
@@ -118,7 +118,7 @@ class Glm4MoE(nn.Module):
def __init__(
self,
config: Glm4MoeConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
):
@@ -234,14 +234,14 @@ class Glm4MoeAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 131072,
head_dim: Optional[int] = None,
head_dim: int | None = None,
rms_norm_eps: float = 1e-05,
qkv_bias: bool = False,
use_qk_norm: bool = False,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -334,8 +334,8 @@ class Glm4MoeDecoderLayer(nn.Module):
def __init__(
self,
config: Glm4MoeConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
) -> None:
@@ -395,7 +395,7 @@ class Glm4MoeDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
@@ -462,9 +462,9 @@ class Glm4MoeModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -704,9 +704,9 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -715,7 +715,7 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -729,7 +729,7 @@ class Glm4MoeForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
def get_spec_layer_idx_from_weight_name(
config: Glm4MoeConfig, weight_name: str
) -> Optional[int]:
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
+8 -9
View File
@@ -24,7 +24,6 @@
"""Inference-only GLM-4.5 MTP model compatible with HuggingFace weights."""
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -52,7 +51,7 @@ class SharedHead(nn.Module):
self,
config: PretrainedConfig,
prefix: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -72,8 +71,8 @@ class Glm4MoeMultiTokenPredictorLayer(nn.Module):
self,
config: PretrainedConfig,
prefix: str,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -94,7 +93,7 @@ class Glm4MoeMultiTokenPredictorLayer(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
@@ -149,7 +148,7 @@ class Glm4MoeMultiTokenPredictor(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
@@ -192,8 +191,8 @@ class Glm4MoeMTP(nn.Module, SupportsPP):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
@@ -205,7 +204,7 @@ class Glm4MoeMTP(nn.Module, SupportsPP):
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+22 -22
View File
@@ -8,7 +8,7 @@
import itertools
from argparse import Namespace
from collections.abc import Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal
import torch
from torch import nn
@@ -109,7 +109,7 @@ class EVA2CLIPAttention(nn.Module):
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -152,7 +152,7 @@ class EVA2CLIPMLP(nn.Module):
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -182,7 +182,7 @@ class EVA2CLIPTransformerLayer(nn.Module):
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -211,7 +211,7 @@ class EVA2CLIPTransformer(nn.Module):
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -237,7 +237,7 @@ class EVA2CLIPGLU(nn.Module):
self,
config,
in_features,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
"""
@@ -317,7 +317,7 @@ class EVA2CLIPModel(nn.Module):
def __init__(
self,
config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -416,9 +416,9 @@ class GLM4VProcessor:
def __call__(
self,
text: Optional[Union[TextInput, list[TextInput]]] = None,
images: Optional[Union[ImageInput, list[ImageInput]]] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
text: TextInput | list[TextInput] | None = None,
images: ImageInput | list[ImageInput] | None = None,
return_tensors: str | TensorType | None = None,
) -> BatchFeature:
if text is None:
text = []
@@ -458,7 +458,7 @@ class GLM4VProcessingInfo(BaseProcessingInfo):
**kwargs,
)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": 1}
def get_num_image_tokens(self) -> int:
@@ -487,7 +487,7 @@ class GLM4VDummyInputsBuilder(BaseDummyInputsBuilder[GLM4VProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
hf_config = self.info.get_hf_config()
vision_config = hf_config.vision_config
@@ -578,7 +578,7 @@ class GLM4VForCausalLM(
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|begin_of_image|><|endoftext|><|end_of_image|>"
@@ -601,7 +601,7 @@ class GLM4VForCausalLM(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[GLMVImagePixelInputs]:
) -> GLMVImagePixelInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
if pixel_values is not None:
@@ -624,12 +624,12 @@ class GLM4VForCausalLM(
cls,
input_tokens: list[int],
hf_config: PretrainedConfig,
image_grid_thw: Union[list[list[int]], torch.Tensor],
video_grid_thw: Union[list[list[int]], torch.Tensor],
image_grid_thw: list[list[int]] | torch.Tensor,
video_grid_thw: list[list[int]] | torch.Tensor,
context_len: int = 0,
seq_len: Optional[int] = None,
second_per_grid_ts: Optional[list[float]] = None,
audio_feature_lengths: Optional[torch.Tensor] = None,
seq_len: int | None = None,
second_per_grid_ts: list[float] | None = None,
audio_feature_lengths: torch.Tensor | None = None,
use_audio_in_video: bool = False,
) -> tuple[torch.Tensor, int]:
"""Get mrope input positions and delta value for GLM4V."""
@@ -780,10 +780,10 @@ class GLM4VForCausalLM(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
+14 -15
View File
@@ -22,7 +22,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -65,8 +64,8 @@ class GPT2Attention(nn.Module):
def __init__(
self,
config: GPT2Config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -118,7 +117,7 @@ class GPT2MLP(nn.Module):
self,
intermediate_size: int,
config: GPT2Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -150,8 +149,8 @@ class GPT2Block(nn.Module):
def __init__(
self,
config: GPT2Config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -221,9 +220,9 @@ class GPT2Model(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor],
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings(input_ids)
@@ -301,9 +300,9 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -312,7 +311,7 @@ class GPT2LMHeadModel(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -367,8 +366,8 @@ class GPT2ForSequenceClassification(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.transformer(
input_ids=input_ids,
+12 -13
View File
@@ -23,7 +23,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -62,8 +61,8 @@ class GPTBigCodeAttention(nn.Module):
def __init__(
self,
config: GPTBigCodeConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -133,7 +132,7 @@ class GPTBigMLP(nn.Module):
self,
intermediate_size: int,
config: GPTBigCodeConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -165,8 +164,8 @@ class GPTBigCodeBlock(nn.Module):
def __init__(
self,
config: GPTBigCodeConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -243,9 +242,9 @@ class GPTBigCodeModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings(input_ids)
@@ -326,9 +325,9 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -337,7 +336,7 @@ class GPTBigCodeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+12 -13
View File
@@ -21,7 +21,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -64,8 +63,8 @@ class GPTJAttention(nn.Module):
def __init__(
self,
config: GPTJConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -130,7 +129,7 @@ class GPTJMLP(nn.Module):
self,
intermediate_size: int,
config: GPTJConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
hidden_size = config.n_embd
@@ -157,8 +156,8 @@ class GPTJBlock(nn.Module):
def __init__(
self,
config: GPTJConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -218,9 +217,9 @@ class GPTJModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -322,9 +321,9 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -333,7 +332,7 @@ class GPTJForCausalLM(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states, self.lm_head.bias)
return logits
+12 -13
View File
@@ -21,7 +21,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -61,8 +60,8 @@ class GPTNeoXAttention(nn.Module):
def __init__(
self,
config: GPTNeoXConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -125,7 +124,7 @@ class GPTNeoXMLP(nn.Module):
def __init__(
self,
config: GPTNeoXConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.dense_h_to_4h = ColumnParallelLinear(
@@ -151,8 +150,8 @@ class GPTNeoXLayer(nn.Module):
def __init__(
self,
config: GPTNeoXConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -232,9 +231,9 @@ class GPTNeoXModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -320,9 +319,9 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.gpt_neox(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -331,7 +330,7 @@ class GPTNeoXForCausalLM(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.embed_out, hidden_states)
return logits
+7 -8
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Optional
import torch
import torch.distributed as dist
@@ -49,8 +48,8 @@ class OAIAttention(nn.Module):
def __init__(
self,
config: GptOssConfig,
quant_config: Optional[QuantizationConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: QuantizationConfig | None = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -208,7 +207,7 @@ class TransformerBlock(torch.nn.Module):
self,
hidden_states: torch.Tensor,
positions: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
# Self Attention
if residual is None:
@@ -260,8 +259,8 @@ class GptOssModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -687,8 +686,8 @@ class GptOssForCausalLM(nn.Module, SupportsPP, SupportsEagle3):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
return self.model(input_ids, positions, intermediate_tensors, inputs_embeds)
+15 -15
View File
@@ -26,7 +26,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -73,7 +73,7 @@ class GraniteMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
) -> None:
@@ -113,11 +113,11 @@ class GraniteAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -197,8 +197,8 @@ class GraniteDecoderLayer(nn.Module):
def __init__(
self,
config: GraniteConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -323,11 +323,11 @@ class GraniteModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -480,15 +480,15 @@ class GraniteForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return model_output
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+14 -14
View File
@@ -26,7 +26,7 @@
import math
from collections.abc import Iterable, Mapping
from typing import Annotated, Optional, Union
from typing import Annotated
import torch
import torch.nn.functional as F
@@ -92,7 +92,7 @@ class GraniteSpeechAudioInputs(TensorSchema):
class GraniteSpeechMultiModalProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": 1}
# There is no limit to the maximum number of audio tokens that can be
@@ -196,7 +196,7 @@ class GraniteSpeechDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
audio_overrides = mm_options.get("audio") if mm_options else None
@@ -222,7 +222,7 @@ class GraniteSpeechEncoderProjector(nn.Module):
self,
config: PretrainedConfig,
cache_config: CacheConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -279,7 +279,7 @@ class GraniteSpeechConformerFeedForward(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -479,7 +479,7 @@ class GraniteSpeechCTCEncoder(nn.Module):
self,
config: PretrainedConfig,
prefix: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
self.config = config
@@ -561,7 +561,7 @@ class GraniteSpeechForConditionalGeneration(
}
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("audio"):
return "<|audio|>"
@@ -606,7 +606,7 @@ class GraniteSpeechForConditionalGeneration(
def _parse_and_validate_audio_input(
self,
**kwargs: object,
) -> Optional[GraniteSpeechAudioInputs]:
) -> GraniteSpeechAudioInputs | None:
input_features = kwargs.pop("input_features", None)
input_features_mask = kwargs.pop("input_features_mask", None)
audio_embed_sizes = kwargs.pop("audio_embed_sizes", None)
@@ -763,9 +763,9 @@ class GraniteSpeechForConditionalGeneration(
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[torch.Tensor] = None,
is_multimodal: torch.Tensor | None = None,
# Multi-modal token ID may exceed vocab size
handle_oov_mm_token: bool = True,
) -> torch.Tensor:
@@ -784,10 +784,10 @@ class GraniteSpeechForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
@@ -799,7 +799,7 @@ class GraniteSpeechForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(
+13 -13
View File
@@ -26,7 +26,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional
from typing import Any
import torch
from torch import nn
@@ -79,9 +79,9 @@ class GraniteMoeMoE(nn.Module):
top_k: int,
hidden_size: int,
intermediate_size: int,
params_dtype: Optional[torch.dtype] = None,
quant_config: Optional[QuantizationConfig] = None,
tp_size: Optional[int] = None,
params_dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
tp_size: int | None = None,
is_sequence_parallel=False,
prefix: str = "",
):
@@ -143,10 +143,10 @@ class GraniteMoeAttention(nn.Module):
num_kv_heads: int,
max_position: int = 4096 * 32,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
attention_multiplier: Optional[float] = None,
rope_scaling: dict[str, Any] | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
attention_multiplier: float | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -330,8 +330,8 @@ class GraniteMoeModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -557,15 +557,15 @@ class GraniteMoeForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+16 -17
View File
@@ -4,7 +4,6 @@
# Added by the IBM Team, 2025
from collections.abc import Iterable
from typing import Optional
import torch
from torch import nn
@@ -50,9 +49,9 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
self,
config: GraniteMoeHybridConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -105,7 +104,7 @@ class GraniteMoeHybridMambaDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
residual = hidden_states
@@ -139,9 +138,9 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
self,
config: GraniteMoeHybridConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -183,7 +182,7 @@ class GraniteMoeHybridAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> torch.Tensor:
residual = hidden_states
hidden_states = self.input_layernorm(hidden_states)
@@ -218,9 +217,9 @@ class GraniteMoeHybridAttention(nn.Module):
def __init__(
self,
config: GraniteMoeHybridConfig,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -374,8 +373,8 @@ class GraniteMoeHybridModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -614,8 +613,8 @@ class GraniteMoeHybridForCausalLM(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
hidden_states = self.model(
@@ -627,7 +626,7 @@ class GraniteMoeHybridForCausalLM(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -8,7 +8,6 @@ experts.
from collections.abc import Iterable
from itertools import islice
from typing import Optional
import torch
from torch import nn
@@ -41,7 +40,7 @@ class GraniteMoeSharedMLP(nn.Module):
def __init__(
self,
config: GraniteMoeSharedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -80,8 +79,8 @@ class GraniteMoeSharedDecoderLayer(nn.Module):
def __init__(
self,
config: GraniteMoeSharedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -198,8 +197,8 @@ class GraniteMoeSharedModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -321,15 +320,15 @@ class GraniteMoeSharedForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
return hidden_states
def compute_logits(self, hidden_states: torch.Tensor) -> Optional[torch.Tensor]:
def compute_logits(self, hidden_states: torch.Tensor) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+6 -7
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Set
from typing import Optional, Union
import numpy as np
import torch
@@ -62,7 +61,7 @@ class GritLMMeanPool(nn.Module):
arr: np.ndarray,
target: np.ndarray,
start_idx: int = 0,
end_idx: Optional[int] = None,
end_idx: int | None = None,
) -> int:
"""
Find the first occurrence of `target` in `arr` starting from
@@ -152,8 +151,8 @@ class GritLMMeanPool(nn.Module):
def forward_one(
self,
hidden_states: torch.Tensor,
prompt_len: Optional[torch.Tensor] = None,
instr_len: Optional[torch.Tensor] = None,
prompt_len: torch.Tensor | None = None,
instr_len: torch.Tensor | None = None,
) -> torch.Tensor:
assert prompt_len is None or prompt_len == hidden_states.shape[0], (
"partial prefill not supported with MEAN pooling"
@@ -166,7 +165,7 @@ class GritLMMeanPool(nn.Module):
hidden_states: torch.Tensor,
prompt_lens: torch.Tensor,
instr_lens: torch.Tensor,
) -> Union[list[torch.Tensor], torch.Tensor]:
) -> list[torch.Tensor] | torch.Tensor:
offset = 0
pooled_data = list[torch.Tensor]()
@@ -182,9 +181,9 @@ class GritLMMeanPool(nn.Module):
def forward(
self,
hidden_states: Union[torch.Tensor, list[torch.Tensor]],
hidden_states: torch.Tensor | list[torch.Tensor],
pooling_metadata: PoolingMetadata,
) -> Union[list[torch.Tensor], torch.Tensor]:
) -> list[torch.Tensor] | torch.Tensor:
prompt_lens = get_prompt_lens(hidden_states, pooling_metadata)
instr_lens = torch.tensor(
[
+15 -16
View File
@@ -25,7 +25,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
import torch.nn.functional as F
@@ -86,9 +85,9 @@ class Grok1MoE(nn.Module):
top_k: int,
hidden_size: int,
intermediate_size: int,
params_dtype: Optional[torch.dtype] = None,
quant_config: Optional[QuantizationConfig] = None,
tp_size: Optional[int] = None,
params_dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
tp_size: int | None = None,
prefix: str = "",
):
super().__init__()
@@ -137,8 +136,8 @@ class Grok1Attention(nn.Module):
num_kv_heads: int,
max_position: int = 4096 * 32,
rope_theta: float = 10000,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
config=None, # Added config parameter
) -> None:
@@ -223,8 +222,8 @@ class Grok1DecoderLayer(nn.Module):
def __init__(
self,
config,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -273,7 +272,7 @@ class Grok1DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -351,9 +350,9 @@ class Grok1Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -544,9 +543,9 @@ class Grok1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -555,7 +554,7 @@ class Grok1ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+26 -27
View File
@@ -9,7 +9,6 @@
# Licensed under Apache 2.0 License [see LICENSE for details]
# --------------------------------------------------------
from collections.abc import Mapping, Sequence
from typing import Optional, Union
import torch
from PIL import Image
@@ -67,7 +66,7 @@ def get_h2ovl_target_ratios(
min_num: int,
max_num: int,
*,
prior_aspect_ratio: Optional[tuple[int, int]],
prior_aspect_ratio: tuple[int, int] | None,
) -> list[tuple[int, int]]:
target_ratios = get_internvl_target_ratios(min_num, max_num)
@@ -170,7 +169,7 @@ def _preprocess_image(
min_num: int,
max_num: int,
use_thumbnail: bool,
prior_aspect_ratio: Optional[tuple[int, int]],
prior_aspect_ratio: tuple[int, int] | None,
) -> tuple[torch.Tensor, tuple[int, int]]:
target_ratios = get_h2ovl_target_ratios(
min_num,
@@ -244,10 +243,10 @@ class H2OVLProcessor(BaseInternVLProcessor):
config: PretrainedConfig,
tokenizer: AnyTokenizer,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
use_msac: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_msac: bool | None = None,
) -> None:
super().__init__(
config,
@@ -270,7 +269,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
def get_image_repl(
self,
feature_size: int,
num_patches: Optional[int],
num_patches: int | None,
) -> PromptUpdateDetails[str]:
repl_features = IMG_CONTEXT * feature_size
repl_full = IMG_START + repl_features + IMG_END
@@ -280,10 +279,10 @@ class H2OVLProcessor(BaseInternVLProcessor):
def resolve_min_max_num(
self,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
use_thumbnail: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
) -> tuple[int, int]:
min_dynamic_patch = (
self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
@@ -308,12 +307,12 @@ class H2OVLProcessor(BaseInternVLProcessor):
def resolve_target_ratios(
self,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
use_thumbnail: Optional[bool] = None,
prior_aspect_ratio: Optional[tuple[int, int]] = None,
override_min_num: Optional[int] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
prior_aspect_ratio: tuple[int, int] | None = None,
override_min_num: int | None = None,
) -> list[tuple[int, int]]:
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
@@ -335,7 +334,7 @@ class H2OVLProcessor(BaseInternVLProcessor):
*,
image_width: int,
image_height: int,
use_msac: Optional[bool] = None,
use_msac: bool | None = None,
) -> int:
use_msac = self.use_msac if use_msac is None else use_msac
@@ -385,9 +384,9 @@ class H2OVLProcessor(BaseInternVLProcessor):
def _images_to_pixel_values_lst(
self,
images: list[Image.Image],
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
use_msac = self.use_msac if len(images) == 1 else False
@@ -425,8 +424,8 @@ class H2OVLProcessingInfo(BaseInternVLProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[H2OVLProcessor],
use_msac: Optional[bool] = None,
processor: H2OVLProcessor | None,
use_msac: bool | None = None,
) -> int:
if processor is None:
processor = self.get_hf_processor()
@@ -493,11 +492,11 @@ class H2OVLMultiModalProcessor(BaseInternVLMultiModalProcessor[H2OVLProcessingIn
def _cached_apply_hf_processor(
self,
prompt: Union[str, list[int]],
prompt: str | list[int],
mm_data_items: MultiModalDataItems,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Mapping[str, object],
mm_uuids: Optional[MultiModalUUIDDict] = None,
mm_uuids: MultiModalUUIDDict | None = None,
) -> tuple[list[int], MultiModalProcessingInfo, bool]:
# The processor logic is different for len(images) <= 1 vs > 1
# Since the processing cache assumes that the processor output is
@@ -530,7 +529,7 @@ class H2OVLChatModel(InternVLChatModel):
def _init_vision_model(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
is_mono: bool,
prefix: str,
+23 -23
View File
@@ -27,7 +27,7 @@
import typing
from collections.abc import Callable, Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import regex as re
import torch
@@ -102,7 +102,7 @@ class HunYuanMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
reduce_results: bool = True,
@@ -144,11 +144,11 @@ class HunYuanAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
layer_id: int = -1,
) -> None:
@@ -227,7 +227,7 @@ class HunYuanAttention(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
kv_states: Optional[tuple[torch.Tensor]] = None,
kv_states: tuple[torch.Tensor] | None = None,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
@@ -256,11 +256,11 @@ class HunYuanCrossAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
layer_id: int = -1,
) -> None:
@@ -338,7 +338,7 @@ class HunYuanCrossAttention(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
kv_states: Optional[tuple[torch.Tensor]] = None,
kv_states: tuple[torch.Tensor] | None = None,
) -> torch.Tensor:
assert kv_states is not None
ori_k, v = kv_states # use last layer kv,
@@ -365,7 +365,7 @@ class HunYuanSparseMoeBlock(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
layer_id: int = -1,
prefix: str = "",
enable_eplb: bool = False,
@@ -480,8 +480,8 @@ class HunYuanDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
layer_id: int = -1,
enable_eplb: bool = False,
@@ -577,8 +577,8 @@ class HunYuanDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
kv_states: Optional[tuple[torch.Tensor]] = None,
residual: torch.Tensor | None,
kv_states: tuple[torch.Tensor] | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -654,11 +654,11 @@ class HunYuanModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -960,9 +960,9 @@ class HunyuanV1ModelBase(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -971,7 +971,7 @@ class HunyuanV1ModelBase(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -6,7 +6,7 @@ from collections import defaultdict
from collections.abc import Iterable, Mapping, Sequence
from functools import partial
from itertools import accumulate
from typing import Annotated, Any, Literal, Optional, Union
from typing import Annotated, Any, Literal
import numpy as np
import torch
@@ -115,13 +115,13 @@ class HCXVisionProcessingInfo(BaseProcessingInfo):
def get_vision_encoder_info(self):
return get_vision_encoder_info(self.get_hf_config())
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
def get_num_image_tokens(
self,
*,
vision_query_length: Union[int, list[int]],
vision_query_length: int | list[int],
) -> int:
if isinstance(vision_query_length, int):
return vision_query_length
@@ -131,7 +131,7 @@ class HCXVisionProcessingInfo(BaseProcessingInfo):
def get_num_video_tokens(
self,
*,
vision_query_length: Union[int, list[int]],
vision_query_length: int | list[int],
) -> int:
if isinstance(vision_query_length, int):
return vision_query_length
@@ -166,7 +166,7 @@ class HCXVisionDummyInputsBuilder(BaseDummyInputsBuilder[HCXVisionProcessingInfo
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -346,7 +346,7 @@ def _build_hcxvision_hf_processor(
info: HCXVisionProcessingInfo,
dummy_inputs: BaseDummyInputsBuilder[HCXVisionProcessingInfo],
*,
cache: Optional[BaseMultiModalProcessorCache] = None,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor:
if isinstance(info, HCXVisionProcessingInfo):
return HCXVisionMultiModalProcessor(
@@ -360,12 +360,12 @@ def _build_hcxvision_hf_processor(
def init_vision_tower_for_hcxvision(
vision_config,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
use_nth_layer: Optional[int] = None,
require_post_norm: Optional[bool] = None,
use_nth_layer: int | None = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> Union[CLIPVisionModel, SiglipVisionModel]:
) -> CLIPVisionModel | SiglipVisionModel:
num_hidden_layers = vision_config.num_hidden_layers
if not isinstance(use_nth_layer, int):
pass
@@ -473,8 +473,8 @@ class HCXVisionCAbstractor(nn.Module):
def forward(
self,
x: torch.Tensor,
num_queries_vis_abstractors: Optional[list[list[int]]] = None,
num_grids: Optional[list[int]] = None,
num_queries_vis_abstractors: list[list[int]] | None = None,
num_grids: list[int] | None = None,
) -> torch.Tensor:
if self.prenorm is not None:
x = self.prenorm(x)
@@ -493,8 +493,8 @@ class HCXVisionCAbstractor(nn.Module):
def _forward(
self,
x: torch.Tensor,
num_queries_vis_abstractors: Optional[list[list[int]]] = None,
num_grids: Optional[list[int]] = None,
num_queries_vis_abstractors: list[list[int]] | None = None,
num_grids: list[int] | None = None,
) -> torch.Tensor:
# x: [B, L, dim]
B, L, dim = x.shape
@@ -515,8 +515,8 @@ class HCXVisionCAbstractor(nn.Module):
def _forward_adaptive_num_query(
self,
x: torch.Tensor,
num_queries_vis_abstractors: Optional[list[list[int]]] = None,
num_grids: Optional[list[int]] = None,
num_queries_vis_abstractors: list[list[int]] | None = None,
num_grids: list[int] | None = None,
) -> list[torch.Tensor]:
# self.net is consisted by 3 layers (s1, sampler, s2)
assert len(self.net) == 3
@@ -604,7 +604,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
*,
vllm_config: VllmConfig,
prefix: str = "",
**kwargs: Optional[Any],
**kwargs: Any | None,
) -> None:
super().__init__()
@@ -662,7 +662,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
# self.reduction = self._init_reduction_type(use_sum_loss)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return IMAGE_TOKEN
if modality.startswith("video"):
@@ -673,7 +673,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_image_input(
self,
**kwargs: object,
) -> Optional[HCXVisionImageInputs]:
) -> HCXVisionImageInputs | None:
pixel_values_images = kwargs.pop("pixel_values_images", None)
if pixel_values_images is None:
@@ -689,7 +689,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_video_input(
self,
**kwargs: object,
) -> Optional[HCXVisionVideoInputs]:
) -> HCXVisionVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
if pixel_values_videos is None:
@@ -762,10 +762,10 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
@@ -946,7 +946,7 @@ class HCXVisionForCausalLM(nn.Module, SupportsMultiModal, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(
@@ -1062,7 +1062,7 @@ def select_best_resolution(original_size: tuple, possible_resolutions: list) ->
def get_anyres_image_grid_shape(
image_size: tuple[int, int],
grid_pinpoints: Union[str, list[tuple[int, int]]],
grid_pinpoints: str | list[tuple[int, int]],
patch_size: int,
) -> tuple[int, int]:
possible_resolutions = (
@@ -19,7 +19,6 @@
"""PyTorch Idefics2 model."""
from collections.abc import Iterable
from typing import Optional
import torch
from torch import nn
@@ -77,7 +76,7 @@ class Idefics2VisionEmbeddings(nn.Module):
self,
pixel_values: torch.FloatTensor,
patch_attention_mask: torch.BoolTensor,
tgt_sizes: Optional[torch.IntTensor] = None,
tgt_sizes: torch.IntTensor | None = None,
) -> torch.Tensor:
batch_size, _, max_im_h, max_im_w = pixel_values.shape
target_dtype = self.patch_embedding.weight.dtype
@@ -124,7 +123,7 @@ class Idefics2VisionAttention(nn.Module):
def __init__(
self,
config: Idefics2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -185,7 +184,7 @@ class Idefics2VisionMLP(nn.Module):
def __init__(
self,
config: Idefics2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -220,7 +219,7 @@ class Idefics2EncoderLayer(nn.Module):
def __init__(
self,
config: Idefics2Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -275,9 +274,9 @@ class Idefics2Encoder(nn.Module):
def __init__(
self,
config: Idefics2Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
num_hidden_layers_override: int | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -326,9 +325,9 @@ class Idefics2VisionTransformer(nn.Module):
def __init__(
self,
config: Idefics2VisionConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
num_hidden_layers_override: int | None = None,
require_post_norm: bool = True,
prefix: str = "",
use_data_parallel: bool = False,
@@ -370,8 +369,8 @@ class Idefics2VisionTransformer(nn.Module):
def forward(
self,
pixel_values,
patch_attention_mask: Optional[torch.BoolTensor] = None,
tgt_sizes: Optional[torch.IntTensor] = None,
patch_attention_mask: torch.BoolTensor | None = None,
tgt_sizes: torch.IntTensor | None = None,
) -> torch.Tensor:
hidden_states = self.embeddings(
pixel_values=pixel_values,
+22 -24
View File
@@ -18,7 +18,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal, TypeAlias
import torch
from torch import nn
@@ -91,14 +91,14 @@ class Idefics3ImageEmbeddingInputs(TensorSchema):
data: Annotated[torch.Tensor, TensorShape("bn", "f", "h")]
ImageInputs = Union[Idefics3ImagePixelInputs, Idefics3ImageEmbeddingInputs]
ImageInputs: TypeAlias = Idefics3ImagePixelInputs | Idefics3ImageEmbeddingInputs
class Idefics3ProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> Idefics3Processor:
return self.ctx.get_hf_processor(Idefics3Processor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def _resize_output_size(
@@ -106,9 +106,9 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
*,
height: int,
width: int,
max_len: Optional[int] = None,
max_len: int | None = None,
min_len: int = 1,
max_size: Optional[int] = None,
max_size: int | None = None,
) -> tuple[int, int]:
# Set default value for max_len if not provided
max_len = max(height, width) if max_len is None else max_len
@@ -165,7 +165,7 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Idefics3Processor],
processor: Idefics3Processor | None,
) -> tuple[int, int]:
if processor is None:
processor = self.get_hf_processor()
@@ -197,7 +197,7 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Idefics3Processor],
processor: Idefics3Processor | None,
) -> int:
grid_w, grid_h = self._get_image_feature_grid_size(
image_width=image_width,
@@ -208,7 +208,7 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
return grid_w * grid_h + 1
def _get_image_token(
self, processor: Optional[Idefics3Processor]
self, processor: Idefics3Processor | None
) -> tuple[str, str, str]:
if processor is None:
processor = self.get_hf_processor()
@@ -223,7 +223,7 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Idefics3Processor],
processor: Idefics3Processor | None,
) -> str:
if processor is None:
processor = self.get_hf_processor()
@@ -269,7 +269,7 @@ class Idefics3ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[Idefics3Processor],
processor: Idefics3Processor | None,
) -> int:
if processor is None:
processor = self.get_hf_processor()
@@ -305,7 +305,7 @@ class Idefics3DummyInputsBuilder(BaseDummyInputsBuilder[Idefics3ProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
hf_processor = self.info.get_hf_processor()
@@ -425,7 +425,7 @@ class Idefics3SimpleMLP(nn.Module):
def __init__(
self,
config: Idefics3Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -448,7 +448,7 @@ class Idefics3Connector(nn.Module):
def __init__(
self,
config: Idefics3Config,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -557,9 +557,9 @@ class Idefics3Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.text_model(
input_ids,
positions,
@@ -590,7 +590,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo
}
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
@@ -621,9 +621,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo
self.lm_head.weight = self.model.text_model.embed_tokens.weight
self.logits_processor = LogitsProcessor(config.text_config.vocab_size)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[ImageInputs]:
def _parse_and_validate_image_input(self, **kwargs: object) -> ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -663,7 +661,7 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo
def _process_image_input(
self,
image_input: ImageInputs,
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
if image_input["type"] == "image_embeds":
return image_input["data"]
@@ -687,10 +685,10 @@ class Idefics3ForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsLo
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
+67 -65
View File
@@ -1,15 +1,13 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping, MutableSequence
from collections.abc import Callable, Iterable, Mapping, MutableSequence
from typing import (
TYPE_CHECKING,
Callable,
ClassVar,
Literal,
Optional,
Protocol,
Union,
TypeAlias,
overload,
runtime_checkable,
)
@@ -34,10 +32,14 @@ if TYPE_CHECKING:
from vllm.config import VllmConfig
from vllm.model_executor.models.utils import WeightsMapper
from vllm.sequence import IntermediateTensors
else:
VllmConfig = object
WeightsMapper = object
IntermediateTensors = object
logger = init_logger(__name__)
MultiModalEmbeddings = Union[list[Tensor], Tensor, tuple[Tensor, ...]]
MultiModalEmbeddings: TypeAlias = list[Tensor] | Tensor | tuple[Tensor, ...]
"""
The output embeddings must be one of the following formats:
@@ -79,7 +81,7 @@ class SupportsMultiModal(Protocol):
"""
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
"""
Get the placeholder text for the `i`th `modality` item in the prompt.
"""
@@ -127,7 +129,7 @@ class SupportsMultiModal(Protocol):
input_ids: Tensor,
get_input_embeddings: Callable[[Tensor], Tensor],
*,
is_multimodal: Optional[Tensor],
is_multimodal: Tensor | None,
handle_oov_mm_token: bool,
) -> Tensor:
if handle_oov_mm_token and is_multimodal is not None:
@@ -145,9 +147,9 @@ class SupportsMultiModal(Protocol):
def get_input_embeddings(
self,
input_ids: Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[Tensor] = None,
is_multimodal: Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> Tensor:
"""
@@ -236,16 +238,16 @@ def supports_multimodal(model: object) -> TypeIs[SupportsMultiModal]: ...
def supports_multimodal(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsMultiModal]], TypeIs[SupportsMultiModal]]:
model: type[object] | object,
) -> TypeIs[type[SupportsMultiModal]] | TypeIs[SupportsMultiModal]:
return getattr(model, "supports_multimodal", False)
def supports_multimodal_raw_input_only(model: Union[type[object], object]) -> bool:
def supports_multimodal_raw_input_only(model: type[object] | object) -> bool:
return getattr(model, "supports_multimodal_raw_input_only", False)
def supports_multimodal_encoder_tp_data(model: Union[type[object], object]) -> bool:
def supports_multimodal_encoder_tp_data(model: type[object] | object) -> bool:
return getattr(model, "supports_encoder_tp_data", False)
@@ -260,8 +262,8 @@ def supports_multimodal_pruning(model: object) -> TypeIs[SupportsMultiModalPruni
def supports_multimodal_pruning(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsMultiModalPruning]], TypeIs[SupportsMultiModalPruning]]:
model: type[object] | object,
) -> TypeIs[type[SupportsMultiModalPruning]] | TypeIs[SupportsMultiModalPruning]:
return getattr(model, "supports_multimodal_pruning", False)
@@ -279,7 +281,7 @@ class SupportsScoreTemplate(Protocol):
"""
@classmethod
def get_score_template(cls, query: str, document: str) -> Optional[str]:
def get_score_template(cls, query: str, document: str) -> str | None:
"""
Generate a full prompt by populating the score template with query and document content.
""" # noqa: E501
@@ -304,8 +306,8 @@ def supports_score_template(model: object) -> TypeIs[SupportsScoreTemplate]: ...
def supports_score_template(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsScoreTemplate]], TypeIs[SupportsScoreTemplate]]:
model: type[object] | object,
) -> TypeIs[type[SupportsScoreTemplate]] | TypeIs[SupportsScoreTemplate]:
return getattr(model, "supports_score_template", False)
@@ -348,8 +350,8 @@ def supports_lora(model: object) -> TypeIs[SupportsLoRA]: ...
def supports_lora(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsLoRA]], TypeIs[SupportsLoRA]]:
model: type[object] | object,
) -> TypeIs[type[SupportsLoRA]] | TypeIs[SupportsLoRA]:
result = _supports_lora(model)
if not result:
@@ -379,7 +381,7 @@ def supports_lora(
return result
def _supports_lora(model: Union[type[object], object]) -> bool:
def _supports_lora(model: type[object] | object) -> bool:
if isinstance(model, type):
return isinstance(model, _SupportsLoRAType)
@@ -404,15 +406,15 @@ class SupportsPP(Protocol):
batch_size: int,
dtype: torch.dtype,
device: torch.device,
) -> "IntermediateTensors":
) -> IntermediateTensors:
"""Called when PP rank > 0 for profiling purposes."""
...
def forward(
self,
*,
intermediate_tensors: Optional["IntermediateTensors"],
) -> Union[Tensor, "IntermediateTensors"]:
intermediate_tensors: IntermediateTensors | None,
) -> IntermediateTensors | None:
"""
Accept [`IntermediateTensors`][vllm.sequence.IntermediateTensors] when
PP rank > 0.
@@ -434,13 +436,13 @@ class _SupportsPPType(Protocol):
batch_size: int,
dtype: torch.dtype,
device: torch.device,
) -> "IntermediateTensors": ...
) -> IntermediateTensors: ...
def forward(
self,
*,
intermediate_tensors: Optional["IntermediateTensors"],
) -> Union[Tensor, "IntermediateTensors"]: ...
intermediate_tensors: IntermediateTensors | None,
) -> Tensor | IntermediateTensors: ...
@overload
@@ -452,8 +454,8 @@ def supports_pp(model: object) -> TypeIs[SupportsPP]: ...
def supports_pp(
model: Union[type[object], object],
) -> Union[bool, TypeIs[type[SupportsPP]], TypeIs[SupportsPP]]:
model: type[object] | object,
) -> bool | TypeIs[type[SupportsPP]] | TypeIs[SupportsPP]:
supports_attributes = _supports_pp_attributes(model)
supports_inspect = _supports_pp_inspect(model)
@@ -487,14 +489,14 @@ def supports_pp(
return supports_attributes and supports_inspect
def _supports_pp_attributes(model: Union[type[object], object]) -> bool:
def _supports_pp_attributes(model: type[object] | object) -> bool:
if isinstance(model, type):
return isinstance(model, _SupportsPPType)
return isinstance(model, SupportsPP)
def _supports_pp_inspect(model: Union[type[object], object]) -> bool:
def _supports_pp_inspect(model: type[object] | object) -> bool:
model_forward = getattr(model, "forward", None)
if not callable(model_forward):
return False
@@ -523,8 +525,8 @@ def has_inner_state(model: type[object]) -> TypeIs[type[HasInnerState]]: ...
def has_inner_state(
model: Union[type[object], object],
) -> Union[TypeIs[type[HasInnerState]], TypeIs[HasInnerState]]:
model: type[object] | object,
) -> TypeIs[type[HasInnerState]] | TypeIs[HasInnerState]:
return getattr(model, "has_inner_state", False)
@@ -550,8 +552,8 @@ def is_attention_free(model: type[object]) -> TypeIs[type[IsAttentionFree]]: ...
def is_attention_free(
model: Union[type[object], object],
) -> Union[TypeIs[type[IsAttentionFree]], TypeIs[IsAttentionFree]]:
model: type[object] | object,
) -> TypeIs[type[IsAttentionFree]] | TypeIs[IsAttentionFree]:
return getattr(model, "is_attention_free", False)
@@ -570,7 +572,7 @@ class IsHybrid(Protocol):
@classmethod
def get_mamba_state_shape_from_config(
cls,
vllm_config: "VllmConfig",
vllm_config: VllmConfig,
use_v1: bool = True,
) -> tuple[tuple[int, int], tuple[int, int, int]]:
"""Calculate shapes for Mamba's convolutional and state caches.
@@ -596,8 +598,8 @@ def is_hybrid(model: type[object]) -> TypeIs[type[IsHybrid]]: ...
def is_hybrid(
model: Union[type[object], object],
) -> Union[TypeIs[type[IsHybrid]], TypeIs[IsHybrid]]:
model: type[object] | object,
) -> TypeIs[type[IsHybrid]] | TypeIs[IsHybrid]:
return getattr(model, "is_hybrid", False)
@@ -688,8 +690,8 @@ def has_noops(model: type[object]) -> TypeIs[type[HasNoOps]]: ...
def has_noops(
model: Union[type[object], object],
) -> Union[TypeIs[type[HasNoOps]], TypeIs[HasNoOps]]:
model: type[object] | object,
) -> TypeIs[type[HasNoOps]] | TypeIs[HasNoOps]:
return getattr(model, "has_noops", False)
@@ -711,23 +713,23 @@ def supports_cross_encoding(model: object) -> TypeIs[SupportsCrossEncoding]: ...
def _supports_cross_encoding(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
model: type[object] | object,
) -> TypeIs[type[SupportsCrossEncoding]] | TypeIs[SupportsCrossEncoding]:
return getattr(model, "supports_cross_encoding", False)
def supports_cross_encoding(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsCrossEncoding]], TypeIs[SupportsCrossEncoding]]:
model: type[object] | object,
) -> TypeIs[type[SupportsCrossEncoding]] | TypeIs[SupportsCrossEncoding]:
return is_pooling_model(model) and _supports_cross_encoding(model)
class SupportsQuant:
"""The interface required for all models that support quantization."""
hf_to_vllm_mapper: ClassVar[Optional["WeightsMapper"]] = None
packed_modules_mapping: ClassVar[Optional[dict[str, list[str]]]] = None
quant_config: Optional[QuantizationConfig] = None
hf_to_vllm_mapper: ClassVar[WeightsMapper | None] = None
packed_modules_mapping: ClassVar[dict[str, list[str]] | None] = None
quant_config: QuantizationConfig | None = None
def __new__(cls, *args, **kwargs) -> Self:
instance = super().__new__(cls)
@@ -749,7 +751,7 @@ class SupportsQuant:
return instance
@staticmethod
def _find_quant_config(*args, **kwargs) -> Optional[QuantizationConfig]:
def _find_quant_config(*args, **kwargs) -> QuantizationConfig | None:
"""Find quant config passed through model constructor args"""
from vllm.config import VllmConfig # avoid circular import
@@ -797,10 +799,10 @@ class SupportsTranscription(Protocol):
audio: np.ndarray,
stt_config: SpeechToTextConfig,
model_config: ModelConfig,
language: Optional[str],
language: str | None,
task_type: Literal["transcribe", "translate"],
request_prompt: str,
to_language: Optional[str],
to_language: str | None,
) -> PromptType:
"""Get the prompt for the ASR model.
The model has control over the construction, as long as it
@@ -813,7 +815,7 @@ class SupportsTranscription(Protocol):
return {k: v for k, v in LANGUAGES.items() if k not in cls.supported_languages}
@classmethod
def validate_language(cls, language: Optional[str]) -> Optional[str]:
def validate_language(cls, language: str | None) -> str | None:
"""
Ensure the language specified in the transcription request
is a valid ISO 639-1 language code. If the request language is
@@ -850,7 +852,7 @@ class SupportsTranscription(Protocol):
audio_duration_s: float,
stt_config: SpeechToTextConfig,
model_config: ModelConfig,
) -> Optional[int]:
) -> int | None:
"""
Map from audio duration to number of audio tokens produced by the ASR
model, without running a forward pass.
@@ -870,8 +872,8 @@ def supports_transcription(model: object) -> TypeIs[SupportsTranscription]: ...
def supports_transcription(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsTranscription]], TypeIs[SupportsTranscription]]:
model: type[object] | object,
) -> TypeIs[type[SupportsTranscription]] | TypeIs[SupportsTranscription]:
return getattr(model, "supports_transcription", False)
@@ -891,8 +893,8 @@ def supports_v0_only(model: object) -> TypeIs[SupportsV0Only]: ...
def supports_v0_only(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsV0Only]], TypeIs[SupportsV0Only]]:
model: type[object] | object,
) -> TypeIs[type[SupportsV0Only]] | TypeIs[SupportsV0Only]:
return getattr(model, "supports_v0_only", False)
@@ -942,8 +944,8 @@ def supports_eagle3(model: object) -> TypeIs[SupportsEagle3]: ...
def supports_eagle3(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsEagle3]], TypeIs[SupportsEagle3]]:
model: type[object] | object,
) -> TypeIs[type[SupportsEagle3]] | TypeIs[SupportsEagle3]:
return isinstance(model, SupportsEagle3)
@@ -964,12 +966,12 @@ class SupportsMRoPE(Protocol):
self,
input_tokens: list[int],
hf_config: PretrainedConfig,
image_grid_thw: Optional[Union[list[list[int]], torch.Tensor]],
video_grid_thw: Optional[Union[list[list[int]], torch.Tensor]],
second_per_grid_ts: Optional[list[float]] = None,
image_grid_thw: list[list[int]] | torch.Tensor | None,
video_grid_thw: list[list[int]] | torch.Tensor | None,
second_per_grid_ts: list[float] | None = None,
context_len: int = 0,
seq_len: Optional[int] = None,
audio_feature_lengths: Optional[torch.Tensor] = None,
seq_len: int | None = None,
audio_feature_lengths: torch.Tensor | None = None,
use_audio_in_video: bool = False,
) -> tuple[torch.Tensor, int]:
"""
@@ -1007,6 +1009,6 @@ def supports_mrope(model: object) -> TypeIs[SupportsMRoPE]: ...
def supports_mrope(
model: Union[type[object], object],
) -> Union[TypeIs[type[SupportsMRoPE]], TypeIs[SupportsMRoPE]]:
model: type[object] | object,
) -> TypeIs[type[SupportsMRoPE]] | TypeIs[SupportsMRoPE]:
return isinstance(model, SupportsMRoPE)
+11 -15
View File
@@ -5,9 +5,7 @@ from typing import (
Any,
ClassVar,
Literal,
Optional,
Protocol,
Union,
overload,
runtime_checkable,
)
@@ -63,12 +61,12 @@ class VllmModel(Protocol[T_co]):
) -> T_co: ...
def _check_vllm_model_init(model: Union[type[object], object]) -> bool:
def _check_vllm_model_init(model: type[object] | object) -> bool:
model_init = model.__init__
return supports_kw(model_init, "vllm_config")
def _check_vllm_model_get_input_embeddings(model: Union[type[object], object]) -> bool:
def _check_vllm_model_get_input_embeddings(model: type[object] | object) -> bool:
model_get_input_embeddings = getattr(model, "get_input_embeddings", None)
if not callable(model_get_input_embeddings):
logger.warning(
@@ -80,7 +78,7 @@ def _check_vllm_model_get_input_embeddings(model: Union[type[object], object]) -
return True
def _check_vllm_model_forward(model: Union[type[object], object]) -> bool:
def _check_vllm_model_forward(model: type[object] | object) -> bool:
model_forward = getattr(model, "forward", None)
if not callable(model_forward):
return False
@@ -108,8 +106,8 @@ def is_vllm_model(model: object) -> TypeIs[VllmModel]: ...
def is_vllm_model(
model: Union[type[object], object],
) -> Union[TypeIs[type[VllmModel]], TypeIs[VllmModel]]:
model: type[object] | object,
) -> TypeIs[type[VllmModel]] | TypeIs[VllmModel]:
return (
_check_vllm_model_init(model)
and _check_vllm_model_get_input_embeddings(model)
@@ -124,7 +122,7 @@ class VllmModelForTextGeneration(VllmModel[T], Protocol[T]):
def compute_logits(
self,
hidden_states: T,
) -> Optional[T]:
) -> T | None:
"""Return `None` if TP rank > 0."""
...
@@ -140,10 +138,8 @@ def is_text_generation_model(model: object) -> TypeIs[VllmModelForTextGeneration
def is_text_generation_model(
model: Union[type[object], object],
) -> Union[
TypeIs[type[VllmModelForTextGeneration]], TypeIs[VllmModelForTextGeneration]
]:
model: type[object] | object,
) -> TypeIs[type[VllmModelForTextGeneration]] | TypeIs[VllmModelForTextGeneration]:
if not is_vllm_model(model):
return False
@@ -190,8 +186,8 @@ def is_pooling_model(model: object) -> TypeIs[VllmModelForPooling]: ...
def is_pooling_model(
model: Union[type[object], object],
) -> Union[TypeIs[type[VllmModelForPooling]], TypeIs[VllmModelForPooling]]:
model: type[object] | object,
) -> TypeIs[type[VllmModelForPooling]] | TypeIs[VllmModelForPooling]:
if not is_vllm_model(model):
return False
@@ -211,5 +207,5 @@ def default_pooling_type(pooling_type: str):
return func
def get_default_pooling_type(model: Union[type[object], object]) -> str:
def get_default_pooling_type(model: type[object] | object) -> str:
return getattr(model, "default_pooling_type", "LAST")
+12 -13
View File
@@ -9,7 +9,6 @@
# --------------------------------------------------------
from collections.abc import Iterable
from functools import partial
from typing import Optional
import torch
import torch.nn as nn
@@ -121,8 +120,8 @@ class InternVisionPatchModel(nn.Module):
def forward(
self,
pixel_values: Optional[torch.Tensor] = None,
pixel_embeds: Optional[torch.Tensor] = None,
pixel_values: torch.Tensor | None = None,
pixel_embeds: torch.Tensor | None = None,
) -> torch.FloatTensor:
if pixel_values is None and pixel_embeds is None:
raise ValueError("You have to specify pixel_values or pixel_embeds")
@@ -144,7 +143,7 @@ class InternParallelAttention(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_dummy_heads: int = 0,
prefix: str = "",
@@ -240,7 +239,7 @@ class InternMLP(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
use_data_parallel: bool = False,
) -> None:
@@ -277,7 +276,7 @@ class InternVisionEncoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_dummy_heads: int = 0,
prefix: str = "",
@@ -312,7 +311,7 @@ class InternVisionEncoderLayer(nn.Module):
def _init_attn(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
num_dummy_heads: int,
prefix: str = "",
@@ -350,9 +349,9 @@ class InternVisionEncoder(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
num_hidden_layers_override: int | None = None,
num_dummy_heads: int = 0,
prefix: str = "",
use_data_parallel: bool = False,
@@ -395,9 +394,9 @@ class InternVisionModel(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
num_hidden_layers_override: int | None = None,
num_dummy_heads: int = 0,
prefix: str = "",
use_data_parallel: bool = False,
@@ -422,8 +421,8 @@ class InternVisionModel(nn.Module):
def forward(
self,
pixel_values: Optional[torch.Tensor] = None,
pixel_embeds: Optional[torch.Tensor] = None,
pixel_values: torch.Tensor | None = None,
pixel_embeds: torch.Tensor | None = None,
) -> torch.FloatTensor:
if pixel_values is None and pixel_embeds is None:
raise ValueError("You have to specify pixel_values or pixel_embeds")
+17 -17
View File
@@ -4,7 +4,7 @@
from collections.abc import Iterable
from functools import partial
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -54,7 +54,7 @@ class InternLM2MLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -92,10 +92,10 @@ class InternLM2Attention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -198,8 +198,8 @@ class InternLMDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -232,7 +232,7 @@ class InternLMDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -291,9 +291,9 @@ class InternLM2Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -359,8 +359,8 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
@@ -370,7 +370,7 @@ class InternLM2ForCausalLM(nn.Module, SupportsPP, SupportsLoRA):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.output, hidden_states)
return logits
@@ -451,9 +451,9 @@ class InternLM2ForRewardModel(InternLM2ForCausalLM):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
+8 -9
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -25,8 +24,8 @@ class InternLM2VEDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -66,8 +65,8 @@ class InternLM2VEDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
visual_token_mask: Optional[torch.Tensor] = None,
residual: torch.Tensor | None,
visual_token_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -107,10 +106,10 @@ class InternLM2VEModel(InternLM2Model):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
visual_token_mask: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
visual_token_mask: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
+19 -23
View File
@@ -7,7 +7,7 @@
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal, TypeAlias
import regex as re
import torch
@@ -111,12 +111,10 @@ class InternS1ImageEmbeddingInputs(TensorSchema):
"""
type: Literal["image_embeds"] = "image_embeds"
data: Annotated[
Union[torch.Tensor, list[torch.Tensor]], TensorShape("ni", "tifs", "hs")
]
data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("ni", "tifs", "hs")]
InternS1ImageInputs = Union[InternS1ImagePixelInputs, InternS1ImageEmbeddingInputs]
InternS1ImageInputs: TypeAlias = InternS1ImagePixelInputs | InternS1ImageEmbeddingInputs
class InternS1VideoPixelInputs(TensorSchema):
@@ -143,12 +141,10 @@ class InternS1VideoEmbeddingInputs(TensorSchema):
"""
type: Literal["video_embeds"] = "video_embeds"
data: Annotated[
Union[torch.Tensor, list[torch.Tensor]], TensorShape("nv", "tvfs", "hs")
]
data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("nv", "tvfs", "hs")]
InternS1VideoInputs = Union[InternS1VideoPixelInputs, InternS1VideoEmbeddingInputs]
InternS1VideoInputs: TypeAlias = InternS1VideoPixelInputs | InternS1VideoEmbeddingInputs
def resolve_interns1_min_max_num(
@@ -190,7 +186,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
)
return hf_processor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
def get_num_image_tokens(
@@ -198,7 +194,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional["GotOcr2ImageProcessorFast"] = None,
processor: GotOcr2ImageProcessorFast | None = None,
) -> int:
if processor is None:
processor = self.get_hf_processor().image_processor
@@ -213,7 +209,7 @@ class InternS1ProcessingInfo(BaseProcessingInfo):
num_image_tokens = self.get_hf_processor().image_seq_length * num_image_patches
return num_image_tokens
def resolve_target_ratios(self, use_thumbnail: Optional[bool] = None):
def resolve_target_ratios(self, use_thumbnail: bool | None = None):
image_processor = self.get_hf_processor().image_processor
min_dynamic_patch = image_processor.min_patches
max_dynamic_patch = image_processor.max_patches
@@ -298,7 +294,7 @@ class InternS1DummyInputsBuilder(BaseDummyInputsBuilder[InternS1ProcessingInfo])
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
target_num_frames = self.info.get_num_frames_with_most_features(
@@ -523,7 +519,7 @@ class InternS1ForConditionalGeneration(
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
# transformers InternVLProcessor uses <IMG_CONTEXT> as the separator
# refer to https://github.com/huggingface/transformers/blob/f90de364c2484c7c325bbe05befdcf487bd75b63/src/transformers/models/internvl/processing_internvl.py#L116
if modality.startswith("image"):
@@ -576,7 +572,7 @@ class InternS1ForConditionalGeneration(
def _init_vision_model(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
prefix: str,
):
@@ -620,7 +616,7 @@ class InternS1ForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[InternS1ImageInputs]:
) -> InternS1ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_num_patches = kwargs.pop("image_num_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -654,7 +650,7 @@ class InternS1ForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[InternS1VideoInputs]:
) -> InternS1VideoInputs | None:
pixel_values_flat_video = kwargs.pop("pixel_values_videos", None)
video_num_patches = kwargs.pop("video_num_patches", None)
video_embeds = kwargs.pop("video_embeds", None)
@@ -688,7 +684,7 @@ class InternS1ForConditionalGeneration(
def _process_vision_input(
self,
image_input: Union[InternS1ImageInputs, InternS1VideoInputs],
image_input: InternS1ImageInputs | InternS1VideoInputs,
) -> tuple[torch.Tensor, ...]:
if (
image_input["type"] == "image_embeds"
@@ -763,9 +759,9 @@ class InternS1ForConditionalGeneration(
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[torch.Tensor] = None,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
@@ -786,8 +782,8 @@ class InternS1ForConditionalGeneration(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> IntermediateTensors:
if intermediate_tensors is not None:
@@ -807,7 +803,7 @@ class InternS1ForConditionalGeneration(
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+10 -11
View File
@@ -8,7 +8,6 @@
# Licensed under The MIT License [see LICENSE for details]
# --------------------------------------------------------
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -139,7 +138,7 @@ class InternS1VisionEmbeddings(nn.Module):
def forward(
self,
pixel_values: torch.Tensor,
bool_masked_pos: Optional[torch.BoolTensor] = None,
bool_masked_pos: torch.BoolTensor | None = None,
) -> torch.Tensor:
_, _, height, width = pixel_values.shape
embeddings, (patch_height, patch_width) = self.patch_embeddings(pixel_values)
@@ -240,7 +239,7 @@ class InternS1VisionMLP(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -274,7 +273,7 @@ class InternS1VisionLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_dummy_heads: int = 0,
prefix: str = "",
@@ -309,7 +308,7 @@ class InternS1VisionLayer(nn.Module):
def _init_attn(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
num_dummy_heads: int,
prefix: str = "",
@@ -337,9 +336,9 @@ class InternS1VisionEncoder(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
num_hidden_layers_override: int | None = None,
num_dummy_heads: int = 0,
prefix: str = "",
):
@@ -376,9 +375,9 @@ class InternS1VisionModel(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
*,
num_hidden_layers_override: Optional[int] = None,
num_hidden_layers_override: int | None = None,
num_dummy_heads: int = 0,
prefix: str = "",
) -> None:
@@ -404,8 +403,8 @@ class InternS1VisionModel(nn.Module):
def forward(
self,
pixel_values: Optional[torch.Tensor] = None,
pixel_embeds: Optional[torch.Tensor] = None,
pixel_values: torch.Tensor | None = None,
pixel_embeds: torch.Tensor | None = None,
) -> torch.FloatTensor:
if pixel_values is None and pixel_embeds is None:
raise ValueError("You have to specify pixel_values or pixel_embeds")
+61 -61
View File
@@ -10,7 +10,7 @@
import os
from abc import ABC, abstractmethod
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Any, Literal, Optional, TypeVar, Union
from typing import Annotated, Any, Literal, TypeAlias, TypeVar
import numpy.typing as npt
import torch
@@ -94,10 +94,10 @@ class InternVLImageEmbeddingInputs(TensorSchema):
"""
type: Literal["image_embeds"]
data: Annotated[Union[torch.Tensor, list[torch.Tensor]], TensorShape("n", "f", "h")]
data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("n", "f", "h")]
InternVLImageInputs = Union[InternVLImagePixelInputs, InternVLImageEmbeddingInputs]
InternVLImageInputs: TypeAlias = InternVLImagePixelInputs | InternVLImageEmbeddingInputs
class InternVLVideoPixelInputs(TensorSchema):
@@ -124,10 +124,10 @@ class InternVLVideoEmbeddingInputs(TensorSchema):
"""
type: Literal["video_embeds"]
data: Annotated[Union[torch.Tensor, list[torch.Tensor]], TensorShape("n", "f", "h")]
data: Annotated[torch.Tensor | list[torch.Tensor], TensorShape("n", "f", "h")]
InternVLVideoInputs = Union[InternVLVideoPixelInputs, InternVLVideoEmbeddingInputs]
InternVLVideoInputs: TypeAlias = InternVLVideoPixelInputs | InternVLVideoEmbeddingInputs
# adapted from https://huggingface.co/OpenGVLab/InternVL2-1B
@@ -349,9 +349,9 @@ class BaseInternVLProcessor(ABC):
config: PretrainedConfig,
tokenizer: AnyTokenizer,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> None:
super().__init__()
@@ -391,17 +391,17 @@ class BaseInternVLProcessor(ABC):
def get_image_repl(
self,
feature_size: int,
num_patches: Optional[int],
num_patches: int | None,
) -> PromptUpdateDetails[str]:
raise NotImplementedError
def resolve_min_max_num(
self,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
use_thumbnail: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
) -> tuple[int, int]:
min_dynamic_patch = (
self.min_dynamic_patch if min_dynamic_patch is None else min_dynamic_patch
@@ -426,10 +426,10 @@ class BaseInternVLProcessor(ABC):
def resolve_target_ratios(
self,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
use_thumbnail: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
use_thumbnail: bool | None = None,
) -> list[tuple[int, int]]:
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
@@ -463,9 +463,9 @@ class BaseInternVLProcessor(ABC):
def _images_to_pixel_values_lst(
self,
images: list[Image.Image],
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=min_dynamic_patch,
@@ -489,9 +489,9 @@ class BaseInternVLProcessor(ABC):
self,
text: list[str],
images: list[Image.Image],
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
) -> tuple[list[str], dict[str, torch.Tensor]]:
if len(images) == 0:
image_inputs = {}
@@ -517,7 +517,7 @@ class BaseInternVLProcessor(ABC):
text = [t.replace("<image>", image_repl.full, 1) for t in text]
return text, image_inputs
def _make_batch_input(self, input_item: Optional[Union[Any, list[Any]]] = None):
def _make_batch_input(self, input_item: Any | list[Any] | None = None):
if input_item is None:
input_item = []
if not isinstance(input_item, list):
@@ -526,12 +526,12 @@ class BaseInternVLProcessor(ABC):
def __call__(
self,
text: Optional[Union[str, list[str]]] = None,
images: Optional[Union[Image.Image, list[Image.Image]]] = None,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
return_tensors: str | TensorType | None = None,
) -> BatchFeature:
text, images = [self._make_batch_input(x) for x in (text, images)]
@@ -563,10 +563,10 @@ class InternVLProcessor(BaseInternVLProcessor):
config: PretrainedConfig,
tokenizer: AnyTokenizer,
*,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
video_token: Optional[str] = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
video_token: str | None = None,
) -> None:
super().__init__(
config=config,
@@ -583,7 +583,7 @@ class InternVLProcessor(BaseInternVLProcessor):
return self.tokenizer.get_vocab()[IMG_CONTEXT]
@property
def video_token_id(self) -> Optional[int]:
def video_token_id(self) -> int | None:
if self.video_token is None:
return None
return self.tokenizer.get_vocab().get(self.video_token, None)
@@ -595,7 +595,7 @@ class InternVLProcessor(BaseInternVLProcessor):
def _videos_to_pixel_values_lst(
self,
videos: list[npt.NDArray],
dynamic_image_size: Optional[bool] = None,
dynamic_image_size: bool | None = None,
) -> list[torch.Tensor]:
min_num, max_num = self.resolve_min_max_num(
min_dynamic_patch=1,
@@ -619,7 +619,7 @@ class InternVLProcessor(BaseInternVLProcessor):
self,
text: list[str],
videos: list[npt.NDArray],
dynamic_image_size: Optional[bool] = None,
dynamic_image_size: bool | None = None,
):
if len(videos) == 0 or not self.supports_video:
video_inputs = {}
@@ -646,13 +646,13 @@ class InternVLProcessor(BaseInternVLProcessor):
def __call__(
self,
text: Optional[Union[str, list[str]]] = None,
images: Optional[Union[Image.Image, list[Image.Image]]] = None,
videos: Optional[Union[npt.NDArray, list[npt.NDArray]]] = None,
min_dynamic_patch: Optional[int] = None,
max_dynamic_patch: Optional[int] = None,
dynamic_image_size: Optional[bool] = None,
return_tensors: Optional[Union[str, TensorType]] = None,
text: str | list[str] | None = None,
images: Image.Image | list[Image.Image] | None = None,
videos: npt.NDArray | list[npt.NDArray] | None = None,
min_dynamic_patch: int | None = None,
max_dynamic_patch: int | None = None,
dynamic_image_size: bool | None = None,
return_tensors: str | TensorType | None = None,
) -> BatchFeature:
text, images, videos = [
self._make_batch_input(x) for x in (text, images, videos)
@@ -681,7 +681,7 @@ class InternVLProcessor(BaseInternVLProcessor):
def get_image_repl(
self,
feature_size: int,
num_patches: Optional[int],
num_patches: int | None,
) -> PromptUpdateDetails[str]:
repl_features = IMG_CONTEXT * feature_size
repl_full = IMG_START + repl_features + IMG_END
@@ -691,7 +691,7 @@ class InternVLProcessor(BaseInternVLProcessor):
def get_video_repl(
self,
feature_size: int,
num_patches: Optional[int] = None,
num_patches: int | None = None,
video_context_token: str = IMG_CONTEXT,
) -> PromptUpdateDetails[str]:
repl_features = video_context_token * self.num_image_token
@@ -711,7 +711,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> BaseInternVLProcessor:
raise NotImplementedError
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_num_image_tokens(
@@ -719,7 +719,7 @@ class BaseInternVLProcessingInfo(BaseProcessingInfo):
*,
image_width: int,
image_height: int,
processor: Optional[BaseInternVLProcessor],
processor: BaseInternVLProcessor | None,
) -> int:
if processor is None:
processor = self.get_hf_processor()
@@ -779,7 +779,7 @@ class BaseInternVLDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
target_width, target_height = self.info.get_image_size_with_most_features()
num_images = mm_counts.get("image", 0)
@@ -901,7 +901,7 @@ class InternVLProcessingInfo(BaseInternVLProcessingInfo):
video_limit = {"video": None} if self.supports_video else {}
return {**super().get_supported_mm_limits(), **video_limit}
def get_video_token(self) -> Optional[str]:
def get_video_token(self) -> str | None:
text_model_type = self.get_hf_config().get_text_config().model_type
video_token_map = {
"qwen2": "<|video_pad|>",
@@ -951,7 +951,7 @@ class InternVLDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
dummy_image = super().get_dummy_mm_data(
seq_len=seq_len, mm_counts=mm_counts, mm_options=mm_options
@@ -1079,7 +1079,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
supports_encoder_tp_data = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
if modality.startswith("video"):
@@ -1149,7 +1149,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
def _init_vision_model(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
is_mono: bool,
prefix: str,
@@ -1217,7 +1217,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[InternVLImageInputs]:
) -> InternVLImageInputs | None:
pixel_values_flat = kwargs.pop("pixel_values_flat", None)
image_num_patches = kwargs.pop("image_num_patches", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -1250,7 +1250,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[InternVLVideoPixelInputs]:
) -> InternVLVideoPixelInputs | None:
pixel_values_flat_video = kwargs.pop("pixel_values_flat_video", None)
video_num_patches = kwargs.pop("video_num_patches", None)
video_embeds = kwargs.pop("image_embeds", None)
@@ -1283,7 +1283,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
def _process_vision_input(
self,
image_input: Union[InternVLImageInputs, InternVLVideoInputs],
image_input: InternVLImageInputs | InternVLVideoInputs,
) -> tuple[torch.Tensor, ...]:
if (
image_input["type"] == "image_embeds"
@@ -1364,9 +1364,9 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[torch.Tensor] = None,
is_multimodal: torch.Tensor | None = None,
handle_oov_mm_token: bool = False,
) -> torch.Tensor:
if multimodal_embeddings is not None and len(multimodal_embeddings) > 0:
@@ -1387,8 +1387,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> IntermediateTensors:
if intermediate_tensors is not None:
@@ -1413,7 +1413,7 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+12 -13
View File
@@ -24,7 +24,6 @@
import math
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -86,8 +85,8 @@ class JAISAttention(nn.Module):
def __init__(
self,
config: JAISConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -147,7 +146,7 @@ class JAISMLP(nn.Module):
self,
intermediate_size: int,
config: JAISConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
):
super().__init__()
hidden_size = config.hidden_size
@@ -194,8 +193,8 @@ class JAISBlock(nn.Module):
def __init__(
self,
config: JAISConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -277,9 +276,9 @@ class JAISModel(nn.Module):
self,
input_ids: torch.Tensor,
position_ids: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[IntermediateTensors, torch.Tensor]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> IntermediateTensors | torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings(input_ids)
@@ -341,9 +340,9 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[IntermediateTensors, torch.Tensor]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> IntermediateTensors | torch.Tensor:
hidden_states = self.transformer(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -352,7 +351,7 @@ class JAISLMHeadModel(nn.Module, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+19 -20
View File
@@ -4,7 +4,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional
import torch
from torch import nn
@@ -54,11 +53,11 @@ class JambaMoE(nn.Module):
def __init__(
self,
config: JambaConfig,
num_experts: Optional[int] = None,
top_k: Optional[int] = None,
params_dtype: Optional[torch.dtype] = None,
tp_size: Optional[int] = None,
quant_config: Optional[QuantizationConfig] = None,
num_experts: int | None = None,
top_k: int | None = None,
params_dtype: torch.dtype | None = None,
tp_size: int | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -111,10 +110,10 @@ class JambaMambaDecoderLayer(nn.Module):
self,
config: JambaConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
is_lora_enabled: Optional[bool] = False,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
is_lora_enabled: bool | None = False,
prefix: str = "",
**kwargs,
) -> None:
@@ -159,7 +158,7 @@ class JambaMambaDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -181,9 +180,9 @@ class JambaAttentionDecoderLayer(nn.Module):
self,
config: JambaConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
**kwargs,
) -> None:
@@ -266,7 +265,7 @@ class JambaAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -348,8 +347,8 @@ class JambaModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -523,8 +522,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP, IsHyb
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
hidden_states = self.model(
@@ -568,7 +567,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP, IsHyb
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+4 -5
View File
@@ -1,7 +1,6 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable, Mapping
from typing import Optional
import torch
import torch.nn as nn
@@ -105,14 +104,14 @@ class JinaVLForSequenceClassification(
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|vision_start|><|image_pad|><|vision_end|>"
raise ValueError("Only image modality is supported")
@classmethod
def get_score_template(cls, query: str, document: str) -> Optional[str]:
def get_score_template(cls, query: str, document: str) -> str | None:
return f"**Document**:\n{document}\n**Query**:\n{query}"
@classmethod
@@ -124,8 +123,8 @@ class JinaVLForSequenceClassification(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> torch.Tensor:
hidden_states = super().forward(
+76 -100
View File
@@ -4,7 +4,7 @@ import math
from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Literal, Optional, TypeVar, Union
from typing import Annotated, Any, Literal, TypeAlias, TypeVar
import numpy as np
import torch
@@ -153,7 +153,7 @@ class KeyeImageEmbeddingInputs(TensorSchema):
image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
KeyeImageInputs = Union[KeyeImagePixelInputs, KeyeImageEmbeddingInputs]
KeyeImageInputs: TypeAlias = KeyeImagePixelInputs | KeyeImageEmbeddingInputs
class KeyeVideoPixelInputs(TensorSchema):
@@ -188,7 +188,7 @@ class KeyeVideoEmbeddingInputs(TensorSchema):
video_grid_thw: Annotated[torch.Tensor, TensorShape("nv", 3)]
KeyeVideoInputs = Union[KeyeVideoPixelInputs, KeyeVideoEmbeddingInputs]
KeyeVideoInputs: TypeAlias = KeyeVideoPixelInputs | KeyeVideoEmbeddingInputs
class KeyeVisionEmbeddings(nn.Module):
@@ -278,15 +278,9 @@ class KeyeVisionEmbeddings(nn.Module):
def forward(
self,
pixel_values: torch.FloatTensor,
position_ids: Optional[torch.Tensor] = None,
image_grid_thw: Optional[
list[
Union[
tuple[int, int, int],
list[tuple[int, int, int]],
]
]
] = None,
position_ids: torch.Tensor | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
interpolate_pos_encoding=False,
) -> torch.Tensor:
if pixel_values.dim() == 4:
@@ -357,7 +351,7 @@ class KeyeSiglipAttention(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -416,10 +410,10 @@ class KeyeSiglipAttention(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
attention_mask: Optional[torch.Tensor] = None,
output_attentions: Optional[bool] = False,
cu_seqlens: Optional[list[torch.Tensor]] = None,
rope_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
attention_mask: torch.Tensor | None = None,
output_attentions: bool | None = False,
cu_seqlens: list[torch.Tensor] | None = None,
rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> torch.Tensor:
qkv, _ = self.qkv_proj(hidden_states)
q, k, v = qkv.split(
@@ -524,8 +518,8 @@ class SigLIPRotaryEmbedding(nn.Module):
class KeyeSiglipEncoderLayer(nn.Module):
def __init__(
self,
config: Union[PretrainedConfig],
quant_config: Optional[QuantizationConfig] = None,
config: PretrainedConfig,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -547,9 +541,9 @@ class KeyeSiglipEncoderLayer(nn.Module):
self,
hidden_states: torch.Tensor,
attention_mask: torch.Tensor,
output_attentions: Optional[bool] = False,
cu_seqlens: Optional[list[torch.Tensor]] = None,
rope_emb: Optional[tuple[torch.Tensor, torch.Tensor]] = None,
output_attentions: bool | None = False,
cu_seqlens: list[torch.Tensor] | None = None,
rope_emb: tuple[torch.Tensor, torch.Tensor] | None = None,
) -> tuple[torch.FloatTensor]:
residual = hidden_states
@@ -577,7 +571,7 @@ class KeyeSiglipEncoder(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -610,22 +604,16 @@ class KeyeSiglipEncoder(nn.Module):
def forward(
self,
inputs_embeds,
attention_mask: Optional[torch.Tensor] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
cu_seqlens: Optional[list[torch.Tensor]] = None,
image_grid_thw: Optional[
list[
Union[
tuple[int, int, int],
list[tuple[int, int, int]],
]
]
] = None,
height_position_ids: Optional[torch.Tensor] = None,
width_position_ids: Optional[torch.Tensor] = None,
use_rope: Optional[bool] = False,
window_size: Optional[bool] = -1,
attention_mask: torch.Tensor | None = None,
output_attentions: bool | None = None,
output_hidden_states: bool | None = None,
cu_seqlens: list[torch.Tensor] | None = None,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
use_rope: bool | None = False,
window_size: bool | None = -1,
vision_or_text: str = "vision",
) -> BaseModelOutput:
device = inputs_embeds.device
@@ -676,7 +664,7 @@ class KeyeSiglipVisionTransformer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -694,29 +682,23 @@ class KeyeSiglipVisionTransformer(nn.Module):
def forward(
self,
pixel_values,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
interpolate_pos_encoding: Optional[bool] = False,
attention_mask: Optional[torch.Tensor] = None,
sample_indices: Optional[torch.Tensor] = None,
image_indices: Optional[torch.Tensor] = None,
position_ids: Optional[torch.Tensor] = None,
height_position_ids: Optional[torch.Tensor] = None,
width_position_ids: Optional[torch.Tensor] = None,
cu_seqlens: Optional[list[torch.Tensor]] = None,
padding_mask: Optional[torch.Tensor] = None,
vision_return_embed_list: Optional[bool] = False,
image_grid_thw: Optional[
list[
Union[
tuple[int, int, int],
list[tuple[int, int, int]],
]
]
] = None,
return_pooler_output: Optional[bool] = True,
use_rope: Optional[bool] = False,
window_size: Optional[bool] = -1,
output_attentions: bool | None = None,
output_hidden_states: bool | None = None,
interpolate_pos_encoding: bool | None = False,
attention_mask: torch.Tensor | None = None,
sample_indices: torch.Tensor | None = None,
image_indices: torch.Tensor | None = None,
position_ids: torch.Tensor | None = None,
height_position_ids: torch.Tensor | None = None,
width_position_ids: torch.Tensor | None = None,
cu_seqlens: list[torch.Tensor] | None = None,
padding_mask: torch.Tensor | None = None,
vision_return_embed_list: bool | None = False,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
return_pooler_output: bool | None = True,
use_rope: bool | None = False,
window_size: bool | None = -1,
) -> BaseModelOutputWithPooling:
hidden_states = self.embeddings(
pixel_values,
@@ -763,7 +745,7 @@ class KeyeSiglipVisionModel(nn.Module):
def __init__(
self,
config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -789,24 +771,18 @@ class KeyeSiglipVisionModel(nn.Module):
def forward(
self,
pixel_values,
sample_indices: Optional[torch.Tensor] = None,
output_attentions: Optional[bool] = None,
output_hidden_states: Optional[bool] = None,
sample_indices: torch.Tensor | None = None,
output_attentions: bool | None = None,
output_hidden_states: bool | None = None,
interpolate_pos_encoding: bool = False,
position_ids: Optional[torch.Tensor] = None,
vision_return_embed_list: Optional[bool] = False,
image_grid_thw: Optional[
list[
Union[
tuple[int, int, int],
list[tuple[int, int, int]],
]
]
] = None,
cu_seqlens: Optional[list[torch.Tensor]] = None,
return_pooler_output: Optional[bool] = True,
use_rope: Optional[bool] = False,
window_size: Optional[bool] = -1,
position_ids: torch.Tensor | None = None,
vision_return_embed_list: bool | None = False,
image_grid_thw: list[tuple[int, int, int] | list[tuple[int, int, int]]]
| None = None,
cu_seqlens: list[torch.Tensor] | None = None,
return_pooler_output: bool | None = True,
use_rope: bool | None = False,
window_size: bool | None = -1,
) -> BaseModelOutputWithPooling:
return self.vision_model(
pixel_values=pixel_values,
@@ -893,7 +869,7 @@ class Projector(nn.Module):
self,
text_config: PretrainedConfig,
vision_config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -927,9 +903,9 @@ class Projector(nn.Module):
def forward(
self,
image_features: Union[torch.Tensor, list[torch.Tensor]],
image_features: torch.Tensor | list[torch.Tensor],
image_grid_thw: list[tuple[int, int, int]],
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
m1, m2 = self.merge_kernel_size
if isinstance(image_features, (list, tuple)):
processed_features = list()
@@ -988,7 +964,7 @@ def _keye_field_config(
class KeyeMultiModalDataParser(MultiModalDataParser):
def _parse_image_data(
self,
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any]:
if isinstance(data, dict):
return DictEmbeddingItems(
@@ -1005,7 +981,7 @@ class KeyeMultiModalDataParser(MultiModalDataParser):
def _parse_video_data(
self,
data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any]:
if isinstance(data, dict):
return DictEmbeddingItems(
@@ -1033,7 +1009,7 @@ class KeyeProcessingInfo(BaseProcessingInfo):
def get_supported_mm_limits(
self,
) -> Mapping[str, Optional[int]]:
) -> Mapping[str, int | None]:
return {"image": None, "video": None}
def get_mm_max_tokens_per_item(
@@ -1200,7 +1176,7 @@ class KeyeBaseDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -1303,7 +1279,7 @@ class BaseKeyeModule(nn.Module):
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|vision_start|><|image_pad|><|vision_end|>"
if modality.startswith("video"):
@@ -1348,7 +1324,7 @@ class BaseKeyeModule(nn.Module):
self,
text_config: PretrainedConfig,
vision_config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> nn.Module:
raise ValueError("Need projector")
@@ -1403,8 +1379,8 @@ class BaseKeyeModule(nn.Module):
self,
video_type: Literal["video_embeds", "pixel_values_videos"],
video_grid_thw: list[torch.Tensor],
pixel_values_videos: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, list[torch.Tensor]]:
pixel_values_videos: torch.Tensor | None = None,
) -> torch.Tensor | list[torch.Tensor]:
siglip_position_ids = list()
video_grid_hws = list()
sample_indices = list()
@@ -1473,7 +1449,7 @@ class BaseKeyeModule(nn.Module):
def get_multimodal_embeddings(
self, **kwargs: object
) -> Optional[MultiModalEmbeddings]:
) -> MultiModalEmbeddings | None:
modalities = self._parse_and_validate_multimodal_inputs(**kwargs)
if not modalities:
return None
@@ -1495,10 +1471,10 @@ class BaseKeyeModule(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for Keye-VL.
Args:
@@ -1527,7 +1503,7 @@ class BaseKeyeModule(nn.Module):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -1555,14 +1531,14 @@ class KeyeForConditionalGeneration(
self,
text_config: PretrainedConfig,
vision_config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> nn.Module:
return Projector(text_config, vision_config, quant_config, prefix)
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[KeyeImageInputs]:
) -> KeyeImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
@@ -1586,7 +1562,7 @@ class KeyeForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[KeyeVideoInputs]:
) -> KeyeVideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
+23 -19
View File
@@ -3,7 +3,7 @@
import itertools
from collections.abc import Mapping, Sequence
from functools import partial
from typing import Annotated, Any, Literal, Optional, Union
from typing import Annotated, Any, Literal, TypeAlias
import numpy as np
import torch
@@ -73,7 +73,7 @@ def split_thw(grid_thw: torch.Tensor) -> torch.Tensor:
def get_num_patches(
grid_thw: torch.Tensor, num_frames: Union[list[int], torch.Tensor]
grid_thw: torch.Tensor, num_frames: list[int] | torch.Tensor
) -> list[int]:
"""
Return num_patches per video.
@@ -153,7 +153,9 @@ class KeyeVL1_5ImageEmbeddingInputs(TensorSchema):
image_grid_thw: Annotated[torch.Tensor, TensorShape("ni", 3)]
KeyeVL1_5ImageInputs = Union[KeyeVL1_5ImagePixelInputs, KeyeVL1_5ImageEmbeddingInputs]
KeyeVL1_5ImageInputs: TypeAlias = (
KeyeVL1_5ImagePixelInputs | KeyeVL1_5ImageEmbeddingInputs
)
class KeyeVL1_5VideoPixelInputs(TensorSchema):
@@ -191,7 +193,9 @@ class KeyeVL1_5VideoEmbeddingInputs(TensorSchema):
num_frames: torch.Tensor
KeyeVL1_5VideoInputs = Union[KeyeVL1_5VideoPixelInputs, KeyeVL1_5VideoEmbeddingInputs]
KeyeVL1_5VideoInputs: TypeAlias = (
KeyeVL1_5VideoPixelInputs | KeyeVL1_5VideoEmbeddingInputs
)
class KeyeVL1_5Projector(nn.Module):
@@ -199,7 +203,7 @@ class KeyeVL1_5Projector(nn.Module):
self,
text_config: PretrainedConfig,
vision_config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -233,9 +237,9 @@ class KeyeVL1_5Projector(nn.Module):
def forward(
self,
image_features: Union[torch.Tensor, tuple[torch.Tensor], list[torch.Tensor]],
image_features: torch.Tensor | tuple[torch.Tensor] | list[torch.Tensor],
image_grid_thw: list[tuple[int, int, int]],
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
m1, m2 = self.merge_kernel_size
if isinstance(image_features, (list, tuple)):
processed_features = list()
@@ -275,7 +279,7 @@ class KeyeVL1_5ProcessingInfo(KeyeProcessingInfo):
def get_supported_mm_limits(
self,
) -> Mapping[str, Optional[int]]:
) -> Mapping[str, int | None]:
return {"image": None, "video": 1}
@@ -327,7 +331,7 @@ def _keye_field_config(
class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
def _parse_image_data(
self,
data: Union[dict[str, torch.Tensor], ModalityData[ImageItem]],
data: dict[str, torch.Tensor] | ModalityData[ImageItem],
) -> ModalityDataItems[Any, Any]:
if isinstance(data, dict):
return DictEmbeddingItems(
@@ -344,7 +348,7 @@ class KeyeVL1_5MultiModalDataParser(MultiModalDataParser):
def _parse_video_data(
self,
data: Union[dict[str, torch.Tensor], ModalityData[VideoItem]],
data: dict[str, torch.Tensor] | ModalityData[VideoItem],
) -> ModalityDataItems[Any, Any]:
if isinstance(data, dict):
return DictEmbeddingItems(
@@ -499,7 +503,7 @@ class KeyeVL1_5ForConditionalGeneration(
self,
text_config: PretrainedConfig,
vision_config: PretrainedConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> nn.Module:
return KeyeVL1_5Projector(text_config, vision_config, quant_config, prefix)
@@ -511,7 +515,7 @@ class KeyeVL1_5ForConditionalGeneration(
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[KeyeVL1_5ImageInputs]:
) -> KeyeVL1_5ImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
image_grid_thw = kwargs.pop("image_grid_thw", None)
@@ -535,7 +539,7 @@ class KeyeVL1_5ForConditionalGeneration(
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[KeyeVL1_5VideoInputs]:
) -> KeyeVL1_5VideoInputs | None:
pixel_values_videos = kwargs.pop("pixel_values_videos", None)
video_embeds = kwargs.pop("video_embeds", None)
video_grid_thw = kwargs.pop("video_grid_thw", None)
@@ -595,19 +599,19 @@ class KeyeVL1_5ForConditionalGeneration(
cls,
input_tokens: list[int],
hf_config: PretrainedConfig,
image_grid_thw: Union[list[list[int]], torch.Tensor],
video_grid_thw: Union[list[list[int]], torch.Tensor],
image_grid_thw: list[list[int]] | torch.Tensor,
video_grid_thw: list[list[int]] | torch.Tensor,
context_len: int = 0,
seq_len: Optional[int] = None,
second_per_grid_ts: Optional[list[float]] = None,
audio_feature_lengths: Optional[torch.Tensor] = None,
seq_len: int | None = None,
second_per_grid_ts: list[float] | None = None,
audio_feature_lengths: torch.Tensor | None = None,
use_audio_in_video: bool = False,
) -> tuple[torch.Tensor, int]:
if isinstance(video_grid_thw, list) and len(video_grid_thw) > 0:
video_grid_thw = video_grid_thw[0]
"""Get mrope input positions and delta value (Keye series)."""
def split_thw(grid_thw: Union[torch.Tensor, list[int]]) -> list[list[int]]:
def split_thw(grid_thw: torch.Tensor | list[int]) -> list[list[int]]:
"""
Split grid_thw along the t dimension.
+10 -10
View File
@@ -46,7 +46,7 @@ import copy
import math
from collections.abc import Iterable, Mapping, Sequence
from dataclasses import dataclass
from typing import Annotated, Any, Literal, Optional, Union
from typing import Annotated, Any, Literal
import torch
from torch import nn
@@ -153,7 +153,7 @@ class KimiVLImagePixelInputs(TensorSchema):
type: Literal["pixel_values"] = "pixel_values"
pixel_values: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("np", 3, "ps", "ps"),
]
@@ -169,7 +169,7 @@ class KimiVLProcessingInfo(BaseProcessingInfo):
def get_hf_config(self):
return self.ctx.get_hf_config(KimiVLConfig)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_num_image_tokens(
@@ -227,7 +227,7 @@ class KimiVLDummyInputsBuilder(BaseDummyInputsBuilder[KimiVLProcessingInfo]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
@@ -305,7 +305,7 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
supports_encoder_tp_data = True
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<|media_start|>image<|media_content|><|media_pad|><|media_end|>"
@@ -370,7 +370,7 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[KimiVLImageInputs]:
) -> KimiVLImageInputs | None:
# image input type must be pixel values now
pixel_values = kwargs.pop("pixel_values", None)
image_grid_hws = kwargs.pop("image_grid_hws", None)
@@ -411,7 +411,7 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def get_language_model(self) -> torch.nn.Module:
return self.language_model
def get_multimodal_embeddings(self, **kwargs: object) -> Optional[NestedTensors]:
def get_multimodal_embeddings(self, **kwargs: object) -> NestedTensors | None:
# Validate the multimodal input keyword arguments
image_input = self._parse_and_validate_image_input(**kwargs)
if image_input is None:
@@ -425,8 +425,8 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> IntermediateTensors:
if intermediate_tensors is not None:
@@ -570,7 +570,7 @@ class KimiVLForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def get_spec_layer_idx_from_weight_name(
config: DeepseekV2Config, weight_name: str
) -> Optional[int]:
) -> int | None:
if hasattr(config, "num_nextn_predict_layers") and (
config.num_nextn_predict_layers > 0
):
+18 -18
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional
from typing import Any
import torch
import torch.nn as nn
@@ -54,8 +54,8 @@ class Lfm2MLP(nn.Module):
ff_dim: int,
multiple_of: int,
auto_adjust_ff_dim: bool,
ffn_dim_multiplier: Optional[float],
quant_config: Optional[QuantizationConfig] = None,
ffn_dim_multiplier: float | None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -98,10 +98,10 @@ class Lfm2Attention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -190,9 +190,9 @@ class Lfm2AttentionDecoderLayer(nn.Module):
self,
config: Lfm2Config,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -240,7 +240,7 @@ class Lfm2AttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
@@ -258,9 +258,9 @@ class Lfm2ShortConvDecoderLayer(nn.Module):
self,
config: Lfm2Config,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -289,7 +289,7 @@ class Lfm2ShortConvDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -365,8 +365,8 @@ class Lfm2Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -532,8 +532,8 @@ class Lfm2ForCausalLM(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
hidden_states = self.model(
+18 -18
View File
@@ -2,7 +2,7 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional
from typing import Any
import torch
import torch.nn as nn
@@ -65,7 +65,7 @@ class Lfm2MoeMlp(nn.Module):
self,
dim: int,
ff_dim: int,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -96,7 +96,7 @@ class Lfm2MoeSparseMoeBlock(nn.Module):
def __init__(
self,
config: Lfm2MoeConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
):
@@ -191,10 +191,10 @@ class Lfm2MoeAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -283,9 +283,9 @@ class Lfm2MoeAttentionDecoderLayer(nn.Module):
self,
config: Lfm2MoeConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
) -> None:
@@ -340,7 +340,7 @@ class Lfm2MoeAttentionDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
@@ -358,9 +358,9 @@ class Lfm2MoeShortConvDecoderLayer(nn.Module):
self,
config: Lfm2MoeConfig,
layer_idx: int,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
) -> None:
@@ -396,7 +396,7 @@ class Lfm2MoeShortConvDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -479,8 +479,8 @@ class Lfm2MoeModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -774,8 +774,8 @@ class Lfm2MoeForCausalLM(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
) -> torch.Tensor:
hidden_states = self.model(
+18 -20
View File
@@ -26,7 +26,7 @@
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -76,7 +76,7 @@ class LlamaMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
prefix: str = "",
reduce_results: bool = True,
@@ -121,12 +121,12 @@ class LlamaAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
bias_o_proj: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
attn_type: str = AttentionType.DECODER,
) -> None:
@@ -236,8 +236,8 @@ class LlamaAttention(nn.Module):
def _init_rotary_emb(
self,
config: LlamaConfig,
rope_scaling: Optional[dict[str, Any]],
quant_config: Optional[QuantizationConfig],
rope_scaling: dict[str, Any] | None,
quant_config: QuantizationConfig | None,
) -> None:
is_neox_style = True
is_gguf = quant_config and quant_config.get_name() == "gguf"
@@ -260,7 +260,7 @@ class LlamaDecoderLayer(nn.Module):
self,
vllm_config: VllmConfig,
prefix: str = "",
config: Optional[LlamaConfig] = None,
config: LlamaConfig | None = None,
) -> None:
super().__init__()
@@ -331,7 +331,7 @@ class LlamaDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
@@ -346,7 +346,7 @@ class LlamaDecoderLayer(nn.Module):
hidden_states = self.mlp(hidden_states)
return hidden_states, residual
def get_quant_config(self, vllm_config: VllmConfig) -> Optional[QuantizationConfig]:
def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
"""Get quantization config for this layer. Override in subclasses."""
return vllm_config.quant_config
@@ -407,13 +407,11 @@ class LlamaModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors],
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[
torch.Tensor, IntermediateTensors, tuple[torch.Tensor, list[torch.Tensor]]
]:
intermediate_tensors: IntermediateTensors | None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -627,9 +625,9 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -638,7 +636,7 @@ class LlamaForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+6 -6
View File
@@ -19,7 +19,7 @@
"""Inference-only LLaMA model compatible with HuggingFace weights."""
from collections.abc import Iterable
from typing import Any, Optional
from typing import Any
import torch
from torch import nn
@@ -148,12 +148,12 @@ class Llama4Attention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
bias: bool = False,
bias_o_proj: bool = False,
cache_config: Optional[CacheConfig] = None,
cache_config: CacheConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -296,7 +296,7 @@ class Llama4DecoderLayer(nn.Module):
self,
vllm_config: VllmConfig,
prefix: str = "",
config: Optional[Llama4TextConfig] = None,
config: Llama4TextConfig | None = None,
) -> None:
super().__init__()
@@ -352,7 +352,7 @@ class Llama4DecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
if residual is None:
+5 -6
View File
@@ -17,7 +17,6 @@
# limitations under the License.
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -49,7 +48,7 @@ class LlamaModel(nn.Module):
vllm_config: VllmConfig,
prefix: str = "",
start_layer_id: int = 0,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.config = vllm_config.speculative_config.draft_model_config.hf_config
@@ -81,10 +80,10 @@ class LlamaModel(nn.Module):
def forward(
self,
input_ids: Optional[torch.Tensor],
input_ids: torch.Tensor | None,
positions: torch.Tensor,
hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if inputs_embeds is None:
inputs_embeds = self.get_input_embeddings(input_ids)
@@ -136,7 +135,7 @@ class LlamaModel(nn.Module):
return loaded_params
def validate_and_update_config(
self, start_layer_id: int, quant_config: Optional[QuantizationConfig] = None
self, start_layer_id: int, quant_config: QuantizationConfig | None = None
) -> None:
# yoco and moe is not supported by draft model yet
assert self.config.yoco_global_kv_layer is None
@@ -193,7 +192,7 @@ class EagleLlama4ForCausalLM(Llama4ForCausalLM):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
return self.model(input_ids, positions, hidden_states, inputs_embeds)
+2 -3
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -28,7 +27,7 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
vllm_config: VllmConfig,
disable_input_layernorm: bool,
prefix: str = "",
config: Optional[LlamaConfig] = None,
config: LlamaConfig | None = None,
) -> None:
super().__init__(vllm_config, prefix=prefix, config=config)
@@ -155,7 +154,7 @@ class EagleLlamaForCausalLM(LlamaForCausalLM):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if inputs_embeds is not None:
raise NotImplementedError(
+8 -9
View File
@@ -2,7 +2,6 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -35,7 +34,7 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
self,
vllm_config: VllmConfig,
prefix: str = "",
config: Optional[LlamaConfig] = None,
config: LlamaConfig | None = None,
layer_idx: int = 0,
) -> None:
super().__init__(vllm_config, prefix=prefix, config=config)
@@ -66,7 +65,7 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
else:
self._residual_norm = self._norm_after_residual
def get_quant_config(self, vllm_config: VllmConfig) -> Optional[QuantizationConfig]:
def get_quant_config(self, vllm_config: VllmConfig) -> QuantizationConfig | None:
"""Use drafter's quantization config instead of verifier's."""
draft_model_config = vllm_config.speculative_config.draft_model_config
draft_load_config = vllm_config.load_config
@@ -96,7 +95,7 @@ class LlamaDecoderLayer(LlamaDecoderLayer):
positions: torch.Tensor,
embeds: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if self.layer_idx == 0:
# First layer: concatenate embeds with hidden_states
@@ -182,7 +181,7 @@ class LlamaModel(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
input_embeds: Optional[torch.Tensor] = None,
input_embeds: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
if input_embeds is None:
input_embeds = self.get_input_embeddings(input_ids)
@@ -268,8 +267,8 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[NestedTensors] = None,
is_multimodal: Optional[torch.Tensor] = None,
multimodal_embeddings: NestedTensors | None = None,
is_multimodal: torch.Tensor | None = None,
) -> torch.Tensor:
return self.model.get_input_embeddings(input_ids)
@@ -278,14 +277,14 @@ class Eagle3LlamaForCausalLM(LlamaForCausalLM):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor]:
return self.model(input_ids, positions, hidden_states, inputs_embeds)
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
if self.draft_id_to_target_id is None:
assert logits.shape[1] == self.config.vocab_size, (
+28 -28
View File
@@ -3,7 +3,7 @@
from abc import abstractmethod
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Optional, Protocol, TypeVar, Union
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
import torch
import torch.nn as nn
@@ -93,7 +93,7 @@ class PixtralHFImagePixelInputs(TensorSchema):
type: Literal["pixel_values_pixtral"] = "pixel_values_pixtral"
pixel_values: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "c", "h", "w", dynamic_dims={"h", "w"}),
]
@@ -110,9 +110,9 @@ class LlavaImageEmbeddingInputs(TensorSchema):
data: Annotated[torch.Tensor, TensorShape("bn", "ifs", "hs")]
LlavaImageInputs = Union[
LlavaImagePixelInputs, PixtralHFImagePixelInputs, LlavaImageEmbeddingInputs
]
LlavaImageInputs: TypeAlias = (
LlavaImagePixelInputs | PixtralHFImagePixelInputs | LlavaImageEmbeddingInputs
)
class LlavaMultiModalProjector(nn.Module):
@@ -122,7 +122,7 @@ class LlavaMultiModalProjector(nn.Module):
text_hidden_size: int,
projector_hidden_act: str,
multimodal_projector_bias: bool,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -154,7 +154,7 @@ class LlavaLikeConfig(Protocol):
vision_config: Final[PretrainedConfig]
image_token_index: Final[int]
vision_feature_select_strategy: Final[str]
vision_feature_layer: Final[Union[int, list[int]]]
vision_feature_layer: Final[int | list[int]]
class LlavaLikeProcessor(Protocol):
@@ -172,7 +172,7 @@ class BaseLlavaProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object) -> LlavaLikeProcessor:
raise NotImplementedError
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None}
def get_num_image_tokens(
@@ -222,7 +222,7 @@ class LlavaDummyInputsBuilder(BaseDummyInputsBuilder[_I]):
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
@@ -406,7 +406,7 @@ def _build_llava_or_pixtral_hf_processor(
info: _I,
dummy_inputs: BaseDummyInputsBuilder[_I],
*,
cache: Optional[BaseMultiModalProcessorCache] = None,
cache: BaseMultiModalProcessorCache | None = None,
) -> BaseMultiModalProcessor:
if isinstance(info, PixtralHFProcessingInfo):
return PixtralHFMultiModalProcessor(
@@ -461,11 +461,11 @@ def _get_layer_index(feature_layer_index: int, num_hidden_layers: int) -> int:
def init_vision_tower_for_llava(
hf_config: LlavaLikeConfig,
quant_config: Optional[QuantizationConfig],
quant_config: QuantizationConfig | None,
*,
require_post_norm: Optional[bool] = None,
require_post_norm: bool | None = None,
prefix: str = "",
) -> Union[CLIPVisionModel, SiglipVisionModel, PixtralHFVisionModel]:
) -> CLIPVisionModel | SiglipVisionModel | PixtralHFVisionModel:
vision_config = hf_config.vision_config
# Initialize the vision tower only up to the deepest required feature layer
@@ -524,7 +524,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
@@ -585,7 +585,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[LlavaImageInputs]:
) -> LlavaImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -619,9 +619,9 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def _image_pixels_to_features(
self,
vision_tower: Union[CLIPVisionModel, SiglipVisionModel, PixtralHFVisionModel],
pixel_values: Union[torch.Tensor, list[torch.Tensor]],
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
vision_tower: CLIPVisionModel | SiglipVisionModel | PixtralHFVisionModel,
pixel_values: torch.Tensor | list[torch.Tensor],
) -> torch.Tensor | tuple[torch.Tensor, ...]:
# NOTE: we skip the step to select the vision feature layer since
# this is already done inside the vision tower
return vision_tower(
@@ -631,8 +631,8 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def _process_image_pixels(
self,
inputs: Union[LlavaImagePixelInputs, PixtralHFImagePixelInputs],
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
inputs: LlavaImagePixelInputs | PixtralHFImagePixelInputs,
) -> torch.Tensor | tuple[torch.Tensor, ...]:
assert self.vision_tower is not None
pixel_values = inputs["pixel_values"]
@@ -642,7 +642,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def _process_image_input(
self,
image_input: LlavaImageInputs,
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
) -> torch.Tensor | tuple[torch.Tensor, ...]:
if image_input["type"] == "image_embeds":
return image_input["data"]
@@ -672,10 +672,10 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for LLaVA-1.5.
One key thing to understand is the `input_ids` already accounts for the
@@ -725,7 +725,7 @@ class LlavaForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
@@ -754,11 +754,11 @@ class MantisProcessingInfo(LlavaProcessingInfo):
class MantisMultiModalProcessor(LlavaMultiModalProcessor):
def apply(
self,
prompt: Union[str, list[int]],
prompt: str | list[int],
mm_data: MultiModalDataDict,
hf_processor_mm_kwargs: Mapping[str, object],
tokenization_kwargs: Optional[Mapping[str, object]] = None,
mm_uuids: Optional[MultiModalUUIDDict] = None,
tokenization_kwargs: Mapping[str, object] | None = None,
mm_uuids: MultiModalUUIDDict | None = None,
) -> MultiModalInputs:
hf_config = self.info.get_hf_config()
image_token_id = hf_config.image_token_index
+17 -15
View File
@@ -3,7 +3,7 @@
from abc import abstractmethod
from collections.abc import Iterable, Mapping
from typing import Annotated, Final, Literal, Optional, Protocol, TypeVar, Union
from typing import Annotated, Final, Literal, Protocol, TypeAlias, TypeVar
import torch
import torch.nn as nn
@@ -55,11 +55,11 @@ class LlavaNextImagePixelInputs(TensorSchema):
type: Literal["pixel_values"] = "pixel_values"
pixel_values: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "np", 3, "h", "w", dynamic_dims={"np"}),
]
image_sizes: Annotated[Optional[torch.Tensor], TensorShape("bn", 2)]
image_sizes: Annotated[torch.Tensor | None, TensorShape("bn", 2)]
# This should be in `(height, width)` format.
@@ -75,7 +75,9 @@ class LlavaNextImageEmbeddingInputs(TensorSchema):
data: Annotated[torch.Tensor, TensorShape("bn", "ifs", "hs")]
LlavaNextImageInputs = Union[LlavaNextImagePixelInputs, LlavaNextImageEmbeddingInputs]
LlavaNextImageInputs: TypeAlias = (
LlavaNextImagePixelInputs | LlavaNextImageEmbeddingInputs
)
class LlavaNextLikeConfig(LlavaLikeConfig, Protocol):
@@ -235,7 +237,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
@@ -294,7 +296,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[LlavaNextImageInputs]:
) -> LlavaNextImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_sizes = kwargs.pop("image_sizes", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -324,7 +326,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def _image_pixels_to_features(
self,
vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
vision_tower: CLIPVisionModel | SiglipVisionModel,
pixel_values: torch.Tensor,
) -> torch.Tensor:
# NOTE: we skip the step to select the vision feature layer since
@@ -424,7 +426,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def _process_image_pixels(
self,
inputs: LlavaNextImagePixelInputs,
) -> Union[torch.Tensor, tuple[torch.Tensor, ...]]:
) -> torch.Tensor | tuple[torch.Tensor, ...]:
assert self.vision_tower is not None
pixel_values = inputs["pixel_values"]
@@ -456,7 +458,7 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def _process_image_input(
self,
image_input: LlavaNextImageInputs,
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
if image_input["type"] == "image_embeds":
return [image_input["data"]]
@@ -491,9 +493,9 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
def get_input_embeddings(
self,
input_ids: torch.Tensor,
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
multimodal_embeddings: MultiModalEmbeddings | None = None,
*,
is_multimodal: Optional[torch.Tensor] = None,
is_multimodal: torch.Tensor | None = None,
# Multi-modal token ID may exceed vocab size
handle_oov_mm_token: bool = True,
) -> torch.Tensor:
@@ -512,10 +514,10 @@ class LlavaNextForConditionalGeneration(nn.Module, SupportsMultiModal, SupportsP
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for LlaVA-NeXT.
One key thing to understand is the `input_ids` already accounts for the
@@ -573,7 +575,7 @@ model_executor.models.llava_next.LlavaNextProcessingInfo.get_num_image_tokens].
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+11 -11
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Literal, Optional, Union
from typing import Annotated, Literal
import torch
import torch.nn as nn
@@ -66,7 +66,7 @@ class LlavaNextVideoPixelInputs(TensorSchema):
type: Literal["pixel_values_videos"] = "pixel_values_videos"
pixel_values_videos: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "f", 3, "h", "w", dynamic_dims={"f"}),
]
@@ -81,7 +81,7 @@ class LlavaNextVideoProcessingInfo(BaseProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(LlavaNextVideoProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"video": 1}
def get_image_size_with_most_features(self) -> ImageSize:
@@ -165,7 +165,7 @@ class LlavaNextVideoDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_videos = mm_counts.get("video", 0)
@@ -313,7 +313,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
if modality.startswith("video"):
@@ -356,7 +356,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[LlavaNextVideoPixelInputs]:
) -> LlavaNextVideoPixelInputs | None:
"""
A legal video input should have the following dimensions:
{
@@ -381,7 +381,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _video_pixels_to_features(
self,
vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
vision_tower: CLIPVisionModel | SiglipVisionModel,
pixel_values: torch.Tensor,
) -> torch.Tensor:
# NOTE: we skip the step to select the vision feature layer since
@@ -433,10 +433,10 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for LlaVA-NeXT-Video.
Args:
input_ids: Flattened (concatenated) input_ids corresponding to a
@@ -455,7 +455,7 @@ class LlavaNextVideoForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+23 -23
View File
@@ -3,7 +3,7 @@
import math
from collections.abc import Iterable, Mapping, Sequence
from typing import Annotated, Final, Literal, Optional, Protocol, Union
from typing import Annotated, Final, Literal, Protocol, TypeAlias
import torch
import torch.nn as nn
@@ -69,7 +69,7 @@ class LlavaOnevisionVideoPixelInputs(TensorSchema):
type: Literal["pixel_values_videos"] = "pixel_values_videos"
pixel_values_videos: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "f", 3, "h", "w", dynamic_dims={"f"}),
]
@@ -90,11 +90,11 @@ class LlavaOnevisionImagePixelInputs(TensorSchema):
type: Literal["pixel_values"] = "pixel_values"
pixel_values: Annotated[
Union[torch.Tensor, list[torch.Tensor]],
torch.Tensor | list[torch.Tensor],
TensorShape("bn", "np", 3, "h", "w", dynamic_dims={"np"}),
]
image_sizes: Annotated[Optional[torch.Tensor], TensorShape("bn", 2)]
image_sizes: Annotated[torch.Tensor | None, TensorShape("bn", 2)]
class LlavaOnevisionImageEmbeddingInputs(TensorSchema):
@@ -113,13 +113,13 @@ class LlavaOnevisionImageEmbeddingInputs(TensorSchema):
]
LlavaOnevisionImageInputs = Union[
LlavaOnevisionImagePixelInputs, LlavaOnevisionImageEmbeddingInputs
]
LlavaOnevisionImageInputs: TypeAlias = (
LlavaOnevisionImagePixelInputs | LlavaOnevisionImageEmbeddingInputs
)
LlavaOnevisionMultiInputs = Union[
LlavaOnevisionImageInputs, LlavaOnevisionVideoPixelInputs
]
LlavaOnevisionMultiInputs: TypeAlias = (
LlavaOnevisionImageInputs | LlavaOnevisionVideoPixelInputs
)
class LlavaOnevisionLikeConfig(LlavaNextLikeConfig, Protocol):
@@ -133,7 +133,7 @@ class LlavaOnevisionProcessingInfo(LlavaNextProcessingInfo):
def get_hf_processor(self, **kwargs: object):
return self.ctx.get_hf_processor(LlavaOnevisionProcessor, **kwargs)
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"image": None, "video": None}
# Based on: https://github.com/huggingface/text-generation-inference/blob/v3.0.1/server/text_generation_server/models/vlm_causal_lm.py#L86
@@ -276,7 +276,7 @@ class LlavaOnevisionDummyInputsBuilder(
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_images = mm_counts.get("image", 0)
num_videos = mm_counts.get("video", 0)
@@ -493,7 +493,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
)
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("image"):
return "<image>"
if modality.startswith("video"):
@@ -531,7 +531,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _parse_and_validate_image_input(
self, **kwargs: object
) -> Optional[LlavaOnevisionImageInputs]:
) -> LlavaOnevisionImageInputs | None:
pixel_values = kwargs.pop("pixel_values", None)
image_sizes = kwargs.pop("image_sizes", None)
image_embeds = kwargs.pop("image_embeds", None)
@@ -560,7 +560,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _parse_and_validate_video_input(
self, **kwargs: object
) -> Optional[LlavaOnevisionVideoPixelInputs]:
) -> LlavaOnevisionVideoPixelInputs | None:
"""
A legal video input should have the following dimensions:
{
@@ -606,7 +606,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _image_pixels_to_features(
self,
vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
vision_tower: CLIPVisionModel | SiglipVisionModel,
pixel_values: torch.Tensor,
) -> torch.Tensor:
# NOTE: we skip the step to select the vision feature layer since
@@ -726,7 +726,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _process_image_pixels(
self,
inputs: LlavaOnevisionImagePixelInputs,
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
assert self.vision_tower is not None
pixel_values = inputs["pixel_values"]
@@ -761,7 +761,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _process_image_input(
self,
image_input: LlavaOnevisionImageInputs,
) -> Union[torch.Tensor, list[torch.Tensor]]:
) -> torch.Tensor | list[torch.Tensor]:
if image_input["type"] == "image_embeds":
return [image_input["data"]]
@@ -788,7 +788,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def _video_pixels_to_features(
self,
vision_tower: Union[CLIPVisionModel, SiglipVisionModel],
vision_tower: CLIPVisionModel | SiglipVisionModel,
pixel_values: torch.Tensor,
) -> torch.Tensor:
# NOTE: we skip the step to select the vision feature layer since
@@ -893,10 +893,10 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
"""Run forward pass for LlaVA-Onevision.
Args:
input_ids: Flattened (concatenated) input_ids corresponding to a
@@ -915,7 +915,7 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal, Supp
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.language_model.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+13 -14
View File
@@ -36,7 +36,6 @@
import typing
from collections.abc import Callable, Iterable
from itertools import islice
from typing import Optional, Union
import torch
from torch import nn
@@ -194,7 +193,7 @@ class FlashMLP(nn.Module):
hidden_size: int,
intermediate_size: int,
hidden_act: str,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
reduce_results: bool = True,
prefix: str = "",
) -> None:
@@ -270,8 +269,8 @@ class LongcatMoe(nn.Module):
top_k: int,
hidden_size: int,
intermediate_size: int,
params_dtype: Optional[torch.dtype] = None,
quant_config: Optional[QuantizationConfig] = None,
params_dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
):
@@ -329,8 +328,8 @@ class FlashDecoderLayer(nn.Module):
self,
vllm_config: VllmConfig,
config: FlashConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
enable_eplb: bool = False,
) -> None:
@@ -415,7 +414,7 @@ class FlashDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
if residual is None:
residual = hidden_states
@@ -506,9 +505,9 @@ class FlashModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -592,9 +591,9 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
hidden_states = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -603,7 +602,7 @@ class LongcatFlashForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -4,7 +4,6 @@
# Adapted from
# https://github.com/vllm-project/vllm/blob/v0.7.3/vllm/model_executor/models/deepseek_mtp.py
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -35,7 +34,7 @@ class LongCatMultiTokenPredictorLayer(nn.Module):
config: PretrainedConfig,
prefix: str,
vllm_config: VllmConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.enorm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
@@ -55,7 +54,7 @@ class LongCatMultiTokenPredictorLayer(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_index: int = 0,
) -> torch.Tensor:
assert inputs_embeds is not None
@@ -78,7 +77,7 @@ class LongCatMultiTokenPredictor(nn.Module):
self,
*,
vllm_config: VllmConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -110,7 +109,7 @@ class LongCatMultiTokenPredictor(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
@@ -155,8 +154,8 @@ class LongCatFlashMTP(nn.Module, SupportsPP):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
hidden_states = self.model(
@@ -168,7 +167,7 @@ class LongCatFlashMTP(nn.Module, SupportsPP):
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
@@ -344,7 +343,7 @@ class LongCatFlashMTP(nn.Module, SupportsPP):
def get_spec_layer_idx_from_weight_name(
self, config: PretrainedConfig, weight_name: str
) -> Optional[int]:
) -> int | None:
if "model.mtp" in weight_name:
return config.num_hidden_layers * 2
return None
+9 -10
View File
@@ -4,7 +4,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional
import torch
from torch import nn
@@ -49,10 +48,10 @@ class MambaDecoderLayer(nn.Module):
def __init__(
self,
config: MambaConfig,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
is_lora_enabled: Optional[bool] = False,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
is_lora_enabled: bool | None = False,
prefix: str = "",
) -> None:
super().__init__()
@@ -83,7 +82,7 @@ class MambaDecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -149,8 +148,8 @@ class MambaModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -244,8 +243,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
hidden_states = self.backbone(
+8 -9
View File
@@ -3,7 +3,6 @@
"""PyTorch MAMBA2 model."""
from collections.abc import Iterable
from typing import Optional
import torch
from torch import nn
@@ -44,9 +43,9 @@ class Mamba2DecoderLayer(nn.Module):
def __init__(
self,
config: MambaConfig,
model_config: Optional[ModelConfig] = None,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
model_config: ModelConfig | None = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -76,7 +75,7 @@ class Mamba2DecoderLayer(nn.Module):
def forward(
self,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
**kwargs,
):
if residual is None:
@@ -142,8 +141,8 @@ class Mamba2Model(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
@@ -277,8 +276,8 @@ class Mamba2ForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs,
):
hidden_states = self.backbone(
+26 -26
View File
@@ -25,8 +25,8 @@
import collections
import collections.abc
from collections.abc import Iterable, Mapping, Sequence
from typing import Any, Callable, Optional, TypedDict, Union, cast
from collections.abc import Callable, Iterable, Mapping, Sequence
from typing import Any, TypeAlias, TypedDict, cast
import numpy as np
import torch
@@ -66,7 +66,7 @@ from vllm.transformers_utils.configs.midashenglm import DashengConfig
from .interfaces import MultiModalEmbeddings, SupportsMultiModal, SupportsPP
from .utils import AutoWeightsLoader, init_vllm_registered_model, maybe_prefix
_Tuple2 = Union[int, tuple[int, int], Sequence[int]]
_Tuple2: TypeAlias = int | tuple[int, int] | Sequence[int]
def _resolve_tuple2(x: _Tuple2) -> tuple[int, int]:
@@ -105,7 +105,7 @@ class AudioPatchEmbed(nn.Module):
patch_stride: _Tuple2 = 16,
in_chans: int = 1,
embed_dim: int = 768,
norm_layer: Optional[Callable] = None,
norm_layer: Callable | None = None,
flatten: bool = False,
):
super().__init__()
@@ -151,9 +151,9 @@ class DashengMlp(nn.Module):
def __init__(
self,
in_features: int,
hidden_features: Optional[int] = None,
out_features: Optional[int] = None,
quant_config: Optional[QuantizationConfig] = None,
hidden_features: int | None = None,
out_features: int | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -186,7 +186,7 @@ class DashengAttention(nn.Module):
dim: int,
num_heads: int = 8,
qkv_bias: bool = False,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -226,7 +226,7 @@ class DashengAttention(nn.Module):
prefix=f"{prefix}.proj",
)
def forward(self, x: torch.Tensor, mask: Optional[torch.Tensor] = None):
def forward(self, x: torch.Tensor, mask: torch.Tensor | None = None):
B, N, C = x.shape
qkv, _ = self.qkv(x)
@@ -253,8 +253,8 @@ class DashengBlock(nn.Module):
num_heads: int,
mlp_ratio: float = 4.0,
qkv_bias: bool = False,
init_values: Optional[float] = None,
quant_config: Optional[QuantizationConfig] = None,
init_values: float | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -285,7 +285,7 @@ class DashengBlock(nn.Module):
def forward(
self,
x: torch.Tensor,
mask: Optional[torch.Tensor] = None,
mask: torch.Tensor | None = None,
) -> torch.Tensor:
x = x + self.ls1(self.attn(self.norm1(x), mask))
x = x + self.ls2(self.mlp(self.norm2(x)))
@@ -349,7 +349,7 @@ class DashengAudioTransformer(nn.Module):
def __init__(
self,
config: DashengConfig,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -393,7 +393,7 @@ class DashengAudioTransformer(nn.Module):
def forward_features(
self,
x: torch.Tensor,
mask: Optional[torch.Tensor] = None,
mask: torch.Tensor | None = None,
) -> torch.Tensor:
t = x.shape[-1]
x = x + self.time_pos_embed[:, :, :, :t]
@@ -418,8 +418,8 @@ class DashengAudioTransformer(nn.Module):
def forward(
self,
x: torch.Tensor,
x_length: Optional[torch.Tensor] = None,
) -> tuple[torch.Tensor, Optional[torch.Tensor]]:
x_length: torch.Tensor | None = None,
) -> tuple[torch.Tensor, torch.Tensor | None]:
x = self.front_end(x)
x = x.to(self.time_pos_embed.dtype)
target_length_in_patches = self.target_length // 4
@@ -462,8 +462,8 @@ class AudioProjectorSubsample(nn.Module):
in_dim: int,
out_dim: int,
downsample_rate=5,
dtype: Optional[torch.dtype] = None,
quant_config: Optional[QuantizationConfig] = None,
dtype: torch.dtype | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
):
super().__init__()
@@ -524,7 +524,7 @@ class MiDashengLMProcessingInfo(BaseProcessingInfo):
feature_extractor = hf_processor.feature_extractor
return feature_extractor
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
return {"audio": None}
def get_min_audio_len(self):
@@ -550,7 +550,7 @@ class MiDashengLMDummyInputsBuilder(BaseDummyInputsBuilder[MiDashengLMProcessing
self,
seq_len: int,
mm_counts: Mapping[str, int],
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
mm_options: Mapping[str, BaseDummyOptions] | None = None,
) -> MultiModalDataDict:
num_audios = mm_counts.get("audio", 0)
@@ -689,7 +689,7 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
}
@classmethod
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
if modality.startswith("audio"):
return "<|audio_bos|><|AUDIO|><|audio_eos|>"
@@ -750,7 +750,7 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
def _parse_and_validate_audio_input(
self, **kwargs: object
) -> Optional[MiDashengLMAudioInputs]:
) -> MiDashengLMAudioInputs | None:
input_values = kwargs.pop("input_values", None)
audio_length = kwargs.pop("audio_length", None)
@@ -820,10 +820,10 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
**kwargs: object,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
if intermediate_tensors is not None:
inputs_embeds = None
elif inputs_embeds is None:
@@ -845,7 +845,7 @@ class MiDashengLMModel(nn.Module, SupportsMultiModal, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.decoder.compute_logits(hidden_states)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+4 -5
View File
@@ -28,7 +28,6 @@
from collections.abc import Iterable
from itertools import islice
from typing import Optional, Union
import torch
import torch.nn as nn
@@ -64,9 +63,9 @@ class MiMoModel(Qwen2Model):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, IntermediateTensors]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -185,7 +184,7 @@ class MiMoForCausalLM(Qwen2ForCausalLM, nn.Module):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
hidden_states = self.model.norm(hidden_states)
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+6 -7
View File
@@ -21,7 +21,6 @@
"""Inference-only MiMo-MTP model."""
from collections.abc import Iterable
from typing import Optional
import torch
import torch.nn as nn
@@ -48,8 +47,8 @@ class MiMoMultiTokenPredictorLayer(nn.Module):
config: PretrainedConfig,
prefix: str,
model_config: ModelConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
@@ -129,7 +128,7 @@ class MiMoMultiTokenPredictor(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
previous_hidden_states: torch.Tensor,
inputs_embeds: Optional[torch.Tensor] = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
if inputs_embeds is None:
@@ -173,8 +172,8 @@ class MiMoMTP(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
spec_step_idx: int = 0,
) -> torch.Tensor:
assert spec_step_idx == 0, "mimo_mtp only support predict one token now"
@@ -187,7 +186,7 @@ class MiMoMTP(nn.Module):
self,
hidden_states: torch.Tensor,
spec_step_idx: int = 0,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
return self.model.compute_logits(hidden_states, self.lm_head, spec_step_idx)
def load_weights(self, weights: Iterable[tuple[str, torch.Tensor]]) -> set[str]:
+19 -23
View File
@@ -27,7 +27,7 @@
import math
from collections.abc import Iterable
from itertools import islice
from typing import Any, Optional, Union
from typing import Any
import torch
from torch import nn
@@ -89,8 +89,8 @@ class MiniCPMMoE(nn.Module):
top_k: int,
hidden_size: int,
intermediate_size: int,
params_dtype: Optional[torch.dtype] = None,
tp_size: Optional[int] = None,
params_dtype: torch.dtype | None = None,
tp_size: int | None = None,
):
super().__init__()
self.tp_size = tp_size or get_tensor_model_parallel_world_size()
@@ -190,7 +190,7 @@ class MiniCPMMLP(nn.Module):
intermediate_size: int,
hidden_act: str,
hidden_act_param: float,
quant_config: Optional[QuantizationConfig] = None,
quant_config: QuantizationConfig | None = None,
) -> None:
super().__init__()
self.gate_up_proj = MergedColumnParallelLinear(
@@ -223,10 +223,10 @@ class MiniCPMAttention(nn.Module):
num_heads: int,
num_kv_heads: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -305,8 +305,8 @@ class MiniCPMDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -362,7 +362,7 @@ class MiniCPMDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
residual = hidden_states
@@ -425,8 +425,8 @@ class MiniCPMModel(nn.Module):
self,
prefix: str,
config: PretrainedConfig,
cache_config: Optional[CacheConfig],
quant_config: Optional[QuantizationConfig],
cache_config: CacheConfig | None,
quant_config: QuantizationConfig | None,
):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
@@ -444,11 +444,9 @@ class MiniCPMModel(nn.Module):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[
torch.Tensor, IntermediateTensors, tuple[torch.Tensor, list[torch.Tensor]]
]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
if get_pp_group().is_first_rank:
if inputs_embeds is not None:
hidden_states = inputs_embeds
@@ -633,11 +631,9 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
intermediate_tensors: Optional[IntermediateTensors] = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> Union[
torch.Tensor, IntermediateTensors, tuple[torch.Tensor, list[torch.Tensor]]
]:
intermediate_tensors: IntermediateTensors | None = None,
inputs_embeds: torch.Tensor | None = None,
) -> torch.Tensor | IntermediateTensors | tuple[torch.Tensor, list[torch.Tensor]]:
model_output = self.model(
input_ids, positions, intermediate_tensors, inputs_embeds
)
@@ -658,7 +654,7 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP, SupportsEagle3):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits
+6 -6
View File
@@ -25,7 +25,7 @@
# limitations under the License.
"""Inference-only MiniCPM3 model compatible with HuggingFace weights."""
from typing import Any, Optional
from typing import Any
import torch
from torch import nn
@@ -63,10 +63,10 @@ class MiniCPM3Attention(nn.Module):
q_lora_rank: int,
kv_lora_rank: int,
rope_theta: float = 10000,
rope_scaling: Optional[dict[str, Any]] = None,
rope_scaling: dict[str, Any] | None = None,
max_position_embeddings: int = 8192,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -214,8 +214,8 @@ class MiniCPM3Model(MiniCPMModel):
self,
prefix: str,
config: PretrainedConfig,
cache_config: Optional[CacheConfig],
quant_config: Optional[QuantizationConfig],
cache_config: CacheConfig | None,
quant_config: QuantizationConfig | None,
):
self.start_layer, self.end_layer, self.layers = make_layers(
config.num_hidden_layers,
+7 -8
View File
@@ -26,7 +26,6 @@
import math
from collections.abc import Iterable
from typing import Optional, Union
import torch
from torch import nn
@@ -61,8 +60,8 @@ class EagleMiniCPMDecoderLayer(nn.Module):
def __init__(
self,
config: PretrainedConfig,
cache_config: Optional[CacheConfig] = None,
quant_config: Optional[QuantizationConfig] = None,
cache_config: CacheConfig | None = None,
quant_config: QuantizationConfig | None = None,
prefix: str = "",
) -> None:
super().__init__()
@@ -118,7 +117,7 @@ class EagleMiniCPMDecoderLayer(nn.Module):
self,
positions: torch.Tensor,
hidden_states: torch.Tensor,
residual: Optional[torch.Tensor],
residual: torch.Tensor | None,
) -> tuple[torch.Tensor, torch.Tensor]:
# Self Attention
residual = hidden_states
@@ -185,8 +184,8 @@ class EagleMiniCPMModel(nn.Module):
self,
prefix: str,
config: PretrainedConfig,
cache_config: Optional[CacheConfig],
quant_config: Optional[QuantizationConfig],
cache_config: CacheConfig | None,
quant_config: QuantizationConfig | None,
start_layer: int,
):
self.eagle_layers = nn.ModuleList(
@@ -210,7 +209,7 @@ class EagleMiniCPMModel(nn.Module):
input_ids: torch.Tensor,
positions: torch.Tensor,
hidden_states: torch.Tensor,
) -> Union[torch.Tensor, IntermediateTensors]:
) -> torch.Tensor | IntermediateTensors:
input_embeds = self.get_input_embeddings(input_ids)
input_embeds = self.input_norm1(input_embeds)
hidden_states = self.input_norm2(hidden_states)
@@ -389,7 +388,7 @@ class EagleMiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
def compute_logits(
self,
hidden_states: torch.Tensor,
) -> Optional[torch.Tensor]:
) -> torch.Tensor | None:
logits = self.logits_processor(self.lm_head, hidden_states)
return logits

Some files were not shown because too many files have changed in this diff Show More