Update Optional[x] -> x | None and Union[x, y] to x | y (#26633)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -4,7 +4,7 @@
|
||||
import math
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from contextlib import nullcontext
|
||||
from typing import Annotated, Literal, Optional, Union, cast
|
||||
from typing import Annotated, Literal, cast
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
@@ -137,7 +137,7 @@ class WhisperAudioInputs(TensorSchema):
|
||||
"""
|
||||
|
||||
input_features: Annotated[
|
||||
Optional[list[torch.Tensor]],
|
||||
list[torch.Tensor] | None,
|
||||
TensorShape("b", "nmb", "t"),
|
||||
]
|
||||
|
||||
@@ -185,8 +185,8 @@ class WhisperAttention(nn.Module):
|
||||
num_heads: int,
|
||||
bias: bool = True,
|
||||
attn_type: AttentionType = AttentionType.DECODER,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -258,7 +258,7 @@ class WhisperAttention(nn.Module):
|
||||
self,
|
||||
embed_dim: int,
|
||||
bias: bool = True,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
self.qkv_proj = QKVParallelLinear(
|
||||
@@ -291,8 +291,8 @@ class WhisperCrossAttention(WhisperAttention):
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
bias: bool = True,
|
||||
cache_config: Optional[CacheConfig] = None,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
cache_config: CacheConfig | None = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__(
|
||||
@@ -309,7 +309,7 @@ class WhisperCrossAttention(WhisperAttention):
|
||||
self,
|
||||
embed_dim: int,
|
||||
bias: bool = True,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
) -> None:
|
||||
self.q_proj = ColumnParallelLinear(
|
||||
@@ -332,7 +332,7 @@ class WhisperCrossAttention(WhisperAttention):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: Optional[torch.Tensor],
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
):
|
||||
q, _ = self.q_proj(hidden_states)
|
||||
|
||||
@@ -357,7 +357,7 @@ class WhisperMLP(nn.Module):
|
||||
embed_dim: int,
|
||||
ffn_dim: int,
|
||||
act_fn: str,
|
||||
quant_config: Optional[QuantizationConfig] = None,
|
||||
quant_config: QuantizationConfig | None = None,
|
||||
prefix: str = "",
|
||||
):
|
||||
super().__init__()
|
||||
@@ -463,7 +463,7 @@ class WhisperDecoderLayer(nn.Module):
|
||||
def forward(
|
||||
self,
|
||||
hidden_states: torch.Tensor,
|
||||
encoder_hidden_states: Optional[torch.Tensor],
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
):
|
||||
residual = hidden_states
|
||||
hidden_states = self.self_attn_layer_norm(hidden_states)
|
||||
@@ -521,7 +521,7 @@ class WhisperEncoder(nn.Module):
|
||||
sinusoids(*self.embed_positions.weight.shape)
|
||||
)
|
||||
|
||||
def forward(self, input_features: Union[torch.Tensor, list[torch.Tensor]]):
|
||||
def forward(self, input_features: torch.Tensor | list[torch.Tensor]):
|
||||
hidden_states = []
|
||||
for features in input_features:
|
||||
embeds = nn.functional.gelu(self.conv1(features))
|
||||
@@ -569,7 +569,7 @@ class WhisperDecoder(nn.Module):
|
||||
self,
|
||||
input_ids,
|
||||
positions: torch.Tensor,
|
||||
encoder_hidden_states: Optional[torch.Tensor],
|
||||
encoder_hidden_states: torch.Tensor | None,
|
||||
):
|
||||
inputs_embeds = self.get_input_embeddings(input_ids)
|
||||
positions = self.embed_positions(positions)
|
||||
@@ -600,8 +600,8 @@ class WhisperModel(nn.Module):
|
||||
|
||||
def forward(
|
||||
self,
|
||||
input_features: Optional[Union[torch.Tensor, list[torch.Tensor]]],
|
||||
input_ids: Optional[torch.Tensor],
|
||||
input_features: torch.Tensor | list[torch.Tensor] | None,
|
||||
input_ids: torch.Tensor | None,
|
||||
positions: torch.Tensor,
|
||||
) -> torch.Tensor:
|
||||
encoder_outputs = self.get_encoder_outputs(input_features)
|
||||
@@ -614,8 +614,8 @@ class WhisperModel(nn.Module):
|
||||
|
||||
def get_encoder_outputs(
|
||||
self,
|
||||
input_features: Optional[Union[torch.Tensor, list[torch.Tensor]]],
|
||||
) -> Optional[torch.Tensor]:
|
||||
input_features: torch.Tensor | list[torch.Tensor] | None,
|
||||
) -> torch.Tensor | None:
|
||||
if input_features is None:
|
||||
return None
|
||||
return self.encoder(input_features)
|
||||
@@ -670,7 +670,7 @@ class WhisperProcessingInfo(BaseProcessingInfo):
|
||||
processor_class.tokenizer_class = tokenizer_class
|
||||
return self.ctx.get_hf_processor(processor_class, **kwargs)
|
||||
|
||||
def get_supported_mm_limits(self) -> Mapping[str, Optional[int]]:
|
||||
def get_supported_mm_limits(self) -> Mapping[str, int | None]:
|
||||
return {"audio": 1}
|
||||
|
||||
def get_feature_extractor(self, **kwargs: object) -> WhisperFeatureExtractor:
|
||||
@@ -693,7 +693,7 @@ class WhisperDummyInputsBuilder(BaseDummyInputsBuilder[WhisperProcessingInfo]):
|
||||
self,
|
||||
seq_len: int,
|
||||
mm_counts: Mapping[str, int],
|
||||
mm_options: Optional[Mapping[str, BaseDummyOptions]] = None,
|
||||
mm_options: Mapping[str, BaseDummyOptions] | None = None,
|
||||
) -> MultiModalDataDict:
|
||||
feature_extractor = self.info.get_feature_extractor()
|
||||
|
||||
@@ -721,9 +721,9 @@ class WhisperMultiModalProcessor(EncDecMultiModalProcessor[WhisperProcessingInfo
|
||||
|
||||
def create_encoder_prompt(
|
||||
self,
|
||||
prompt: Union[str, list[int]],
|
||||
prompt: str | list[int],
|
||||
mm_data: MultiModalDataDict,
|
||||
) -> Union[str, list[int]]:
|
||||
) -> str | list[int]:
|
||||
# Strictly speaking, whisper encoder only accept audio features.
|
||||
# We create a dummy encoder prompt here which will be padded to
|
||||
# num_audio_tokens. So that we can create dummy data from this
|
||||
@@ -804,7 +804,7 @@ class WhisperForConditionalGeneration(
|
||||
supported_languages = ISO639_1_SUPPORTED_LANGS
|
||||
|
||||
@classmethod
|
||||
def validate_language(cls, language: Optional[str]) -> Optional[str]:
|
||||
def validate_language(cls, language: str | None) -> str | None:
|
||||
if language is None:
|
||||
# TODO language should be optional and can be guessed.
|
||||
# For now we default to en. See
|
||||
@@ -823,10 +823,10 @@ class WhisperForConditionalGeneration(
|
||||
audio: np.ndarray,
|
||||
model_config: ModelConfig, # not needed here
|
||||
stt_config: SpeechToTextConfig,
|
||||
language: Optional[str],
|
||||
language: str | None,
|
||||
task_type: Literal["transcribe", "translate"],
|
||||
request_prompt: str,
|
||||
to_language: Optional[str],
|
||||
to_language: str | None,
|
||||
) -> PromptType:
|
||||
if language is None:
|
||||
raise ValueError(
|
||||
@@ -849,7 +849,7 @@ class WhisperForConditionalGeneration(
|
||||
return cast(PromptType, prompt)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> str | None:
|
||||
if modality.startswith("audio"):
|
||||
return None
|
||||
|
||||
@@ -872,7 +872,7 @@ class WhisperForConditionalGeneration(
|
||||
audio_duration_s: float,
|
||||
stt_config: SpeechToTextConfig,
|
||||
model_config: ModelConfig,
|
||||
) -> Optional[int]:
|
||||
) -> int | None:
|
||||
processor = cached_get_processor(model_config.model)
|
||||
hop_length = processor.feature_extractor.hop_length
|
||||
assert hop_length is not None
|
||||
@@ -928,9 +928,9 @@ class WhisperForConditionalGeneration(
|
||||
def get_input_embeddings(
|
||||
self,
|
||||
input_ids: torch.Tensor,
|
||||
multimodal_embeddings: Optional[MultiModalEmbeddings] = None,
|
||||
multimodal_embeddings: MultiModalEmbeddings | None = None,
|
||||
*,
|
||||
is_multimodal: Optional[torch.Tensor] = None,
|
||||
is_multimodal: torch.Tensor | None = None,
|
||||
handle_oov_mm_token: bool = False,
|
||||
) -> torch.Tensor:
|
||||
# This method just returns the decoder sequence embeddings since
|
||||
|
||||
Reference in New Issue
Block a user