[Mypy] Fix mypy typing for P models (#54169)
Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com> Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
co-authored by
Harry Mellor
parent
5093e4844a
commit
784cac7c42
@@ -7,6 +7,7 @@
|
||||
#!/usr/bin/env python3
|
||||
import abc
|
||||
import math
|
||||
import typing
|
||||
from typing import Any, Literal
|
||||
|
||||
import numpy as np
|
||||
@@ -152,7 +153,7 @@ class ConformerEncoderLayer(nn.Module):
|
||||
batch_norm: bool = False,
|
||||
activation: str = "relu",
|
||||
chunk_se: int = 0,
|
||||
chunk_size: int = 18,
|
||||
chunk_size: int | list[int] = 18,
|
||||
conv_activation: str = "relu",
|
||||
conv_glu_type: str = "sigmoid",
|
||||
bias_in_glu: bool = True,
|
||||
@@ -392,6 +393,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
else None
|
||||
)
|
||||
if self.relative_attention_bias_type == "t5":
|
||||
assert relative_attention_bias_args is not None
|
||||
assert self.num_heads % self.attention_group_size == 0, (
|
||||
"attention_group_size must divide n_head"
|
||||
)
|
||||
@@ -405,6 +407,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
else:
|
||||
raise NotImplementedError
|
||||
|
||||
assert self.encoder_embedding_config is not None
|
||||
self.encoder_embedding = MeanVarianceNormLayer(
|
||||
self.encoder_embedding_config["input_size"]
|
||||
)
|
||||
@@ -421,6 +424,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
the cases and return one of them. Torchscript does support that.
|
||||
"""
|
||||
if self.input_layer == "nemo_conv":
|
||||
assert self.nemo_conv_settings is not None
|
||||
# Handle the special causal case
|
||||
subsampling_causal_cond = self.nemo_conv_settings.get(
|
||||
"subsampling", "dw_striding"
|
||||
@@ -431,19 +435,19 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
]
|
||||
is_causal = self.nemo_conv_settings.get("is_causal", False)
|
||||
if is_causal and subsampling_causal_cond:
|
||||
lens_change = (
|
||||
torch.ceil(feature_lens / self.time_reduction).long()
|
||||
if isinstance(feature_lens, Tensor)
|
||||
else math.ceil(feature_lens / self.time_reduction)
|
||||
)
|
||||
feature_lens_remainder = feature_lens % self.time_reduction
|
||||
if isinstance(feature_lens, Tensor):
|
||||
lens_change = torch.ceil(feature_lens / self.time_reduction).long()
|
||||
lens_change[feature_lens_remainder != 1] += 1
|
||||
elif feature_lens_remainder != 1:
|
||||
return lens_change
|
||||
|
||||
lens_change = math.ceil(feature_lens / self.time_reduction)
|
||||
if feature_lens_remainder != 1:
|
||||
lens_change += 1
|
||||
return lens_change
|
||||
ceil_func = math.ceil if isinstance(feature_lens, int) else torch.ceil
|
||||
return ceil_func(feature_lens / self.time_reduction)
|
||||
raise ValueError("unknown input_layer: " + self.input_layer)
|
||||
|
||||
@abc.abstractmethod
|
||||
def forward(self) -> Any:
|
||||
@@ -476,6 +480,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
)
|
||||
left_chunk_train_eff = left_chunk[chunk_size_index]
|
||||
else:
|
||||
assert chunk_size is not None
|
||||
assert isinstance(left_chunk, int)
|
||||
chunk_size_train_eff = chunk_size
|
||||
left_chunk_train_eff = left_chunk
|
||||
|
||||
@@ -516,7 +522,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
seq_len: int,
|
||||
batch_size: int,
|
||||
chunk_size: int | list[int],
|
||||
left_chunk: int | list[int],
|
||||
left_chunk: int | list[int] | None,
|
||||
) -> torch.Tensor:
|
||||
chunk_size_train_eff, left_chunk_train_eff = self._chunk_size_selection(
|
||||
chunk_size, left_chunk
|
||||
@@ -535,6 +541,37 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
|
||||
)
|
||||
return enc_streaming_mask
|
||||
|
||||
@typing.overload
|
||||
def forward_embeddings(
|
||||
self,
|
||||
xs_pad: torch.Tensor,
|
||||
masks: torch.Tensor,
|
||||
chunk_size_nc: None = None,
|
||||
left_chunk_nc: int | list[int] | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]: ...
|
||||
|
||||
@typing.overload
|
||||
def forward_embeddings(
|
||||
self,
|
||||
xs_pad: torch.Tensor,
|
||||
masks: torch.Tensor,
|
||||
chunk_size_nc: int | list[int],
|
||||
left_chunk_nc: int | list[int] | None = None,
|
||||
) -> tuple[
|
||||
torch.Tensor,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor | None,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]: ...
|
||||
|
||||
def forward_embeddings(
|
||||
self,
|
||||
xs_pad: torch.Tensor,
|
||||
@@ -911,6 +948,7 @@ class ConformerEncoder(TransformerEncoderBase):
|
||||
) -> torch.Tensor | None:
|
||||
if self.relative_attention_bias_layer:
|
||||
return self.relative_attention_bias_layer(input_tensor)
|
||||
return None
|
||||
|
||||
def calculate_hs_mask(
|
||||
self, xs_pad: torch.Tensor, device: torch.device, mask: torch.Tensor | None
|
||||
@@ -1148,7 +1186,7 @@ class AudioEmbedding(nn.Module):
|
||||
if kwargs.get("use_qformer", False):
|
||||
qformer_config = kwargs.get("qformer_config", {})
|
||||
qformer_config["attention_dim"] = audio_dim_out
|
||||
self.qformer = WindowQformer(**qformer_config)
|
||||
self.qformer: WindowQformer | None = WindowQformer(**qformer_config)
|
||||
else:
|
||||
self.qformer = None
|
||||
|
||||
@@ -1175,7 +1213,7 @@ class AudioEmbedding(nn.Module):
|
||||
"{i} should be specified outside of the NeMo dictionary"
|
||||
)
|
||||
|
||||
self.conv_ds = NemoConvSubsampling(
|
||||
self.conv_ds: NemoConvSubsampling | None = NemoConvSubsampling(
|
||||
**default_nemo_conv_settings,
|
||||
)
|
||||
else:
|
||||
|
||||
Reference in New Issue
Block a user