[Mypy] Fix mypy typing for P models (#54169)

Signed-off-by: Taneem Ibrahim <taneem.ibrahim@gmail.com>
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
Co-authored-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Taneem Ibrahim
2026-09-04 20:47:11 +00:00
committed by GitHub
co-authored by Harry Mellor
parent 5093e4844a
commit 784cac7c42
9 changed files with 126 additions and 45 deletions
+48 -10
View File
@@ -7,6 +7,7 @@
#!/usr/bin/env python3
import abc
import math
import typing
from typing import Any, Literal
import numpy as np
@@ -152,7 +153,7 @@ class ConformerEncoderLayer(nn.Module):
batch_norm: bool = False,
activation: str = "relu",
chunk_se: int = 0,
chunk_size: int = 18,
chunk_size: int | list[int] = 18,
conv_activation: str = "relu",
conv_glu_type: str = "sigmoid",
bias_in_glu: bool = True,
@@ -392,6 +393,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
else None
)
if self.relative_attention_bias_type == "t5":
assert relative_attention_bias_args is not None
assert self.num_heads % self.attention_group_size == 0, (
"attention_group_size must divide n_head"
)
@@ -405,6 +407,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
else:
raise NotImplementedError
assert self.encoder_embedding_config is not None
self.encoder_embedding = MeanVarianceNormLayer(
self.encoder_embedding_config["input_size"]
)
@@ -421,6 +424,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
the cases and return one of them. Torchscript does support that.
"""
if self.input_layer == "nemo_conv":
assert self.nemo_conv_settings is not None
# Handle the special causal case
subsampling_causal_cond = self.nemo_conv_settings.get(
"subsampling", "dw_striding"
@@ -431,19 +435,19 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
]
is_causal = self.nemo_conv_settings.get("is_causal", False)
if is_causal and subsampling_causal_cond:
lens_change = (
torch.ceil(feature_lens / self.time_reduction).long()
if isinstance(feature_lens, Tensor)
else math.ceil(feature_lens / self.time_reduction)
)
feature_lens_remainder = feature_lens % self.time_reduction
if isinstance(feature_lens, Tensor):
lens_change = torch.ceil(feature_lens / self.time_reduction).long()
lens_change[feature_lens_remainder != 1] += 1
elif feature_lens_remainder != 1:
return lens_change
lens_change = math.ceil(feature_lens / self.time_reduction)
if feature_lens_remainder != 1:
lens_change += 1
return lens_change
ceil_func = math.ceil if isinstance(feature_lens, int) else torch.ceil
return ceil_func(feature_lens / self.time_reduction)
raise ValueError("unknown input_layer: " + self.input_layer)
@abc.abstractmethod
def forward(self) -> Any:
@@ -476,6 +480,8 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
)
left_chunk_train_eff = left_chunk[chunk_size_index]
else:
assert chunk_size is not None
assert isinstance(left_chunk, int)
chunk_size_train_eff = chunk_size
left_chunk_train_eff = left_chunk
@@ -516,7 +522,7 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
seq_len: int,
batch_size: int,
chunk_size: int | list[int],
left_chunk: int | list[int],
left_chunk: int | list[int] | None,
) -> torch.Tensor:
chunk_size_train_eff, left_chunk_train_eff = self._chunk_size_selection(
chunk_size, left_chunk
@@ -535,6 +541,37 @@ class TransformerEncoderBase(abc.ABC, nn.Module):
)
return enc_streaming_mask
@typing.overload
def forward_embeddings(
self,
xs_pad: torch.Tensor,
masks: torch.Tensor,
chunk_size_nc: None = None,
left_chunk_nc: int | list[int] | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor,
torch.Tensor,
]: ...
@typing.overload
def forward_embeddings(
self,
xs_pad: torch.Tensor,
masks: torch.Tensor,
chunk_size_nc: int | list[int],
left_chunk_nc: int | list[int] | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor | None,
torch.Tensor | None,
torch.Tensor,
torch.Tensor,
torch.Tensor,
]: ...
def forward_embeddings(
self,
xs_pad: torch.Tensor,
@@ -911,6 +948,7 @@ class ConformerEncoder(TransformerEncoderBase):
) -> torch.Tensor | None:
if self.relative_attention_bias_layer:
return self.relative_attention_bias_layer(input_tensor)
return None
def calculate_hs_mask(
self, xs_pad: torch.Tensor, device: torch.device, mask: torch.Tensor | None
@@ -1148,7 +1186,7 @@ class AudioEmbedding(nn.Module):
if kwargs.get("use_qformer", False):
qformer_config = kwargs.get("qformer_config", {})
qformer_config["attention_dim"] = audio_dim_out
self.qformer = WindowQformer(**qformer_config)
self.qformer: WindowQformer | None = WindowQformer(**qformer_config)
else:
self.qformer = None
@@ -1175,7 +1213,7 @@ class AudioEmbedding(nn.Module):
"{i} should be specified outside of the NeMo dictionary"
)
self.conv_ds = NemoConvSubsampling(
self.conv_ds: NemoConvSubsampling | None = NemoConvSubsampling(
**default_nemo_conv_settings,
)
else: