Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -300,13 +300,13 @@ class CohereASRMLP(nn.Module):
|
||||
|
||||
|
||||
class FixedPositionalEncoding(nn.Module):
|
||||
"""
|
||||
Fixed positional encoding (embedding layer) from sine and cosine functions
|
||||
"""Fixed positional encoding (embedding layer) from sine and cosine functions
|
||||
of different frequencies according to https://arxiv.org/abs/1706.03762
|
||||
|
||||
Args:
|
||||
hidden_size: size of the embeddings in the model, also known as d_model
|
||||
max_sequence_length: maximum allowed length of the input sequence
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, hidden_size: int, max_sequence_length: int = 512) -> None:
|
||||
@@ -664,7 +664,8 @@ class ConvSubsampling(nn.Module):
|
||||
repeat_num: int = 1,
|
||||
) -> torch.Tensor:
|
||||
"""Calculates the output length of a Tensor passed
|
||||
through a convolution or max pooling layer"""
|
||||
through a convolution or max pooling layer
|
||||
"""
|
||||
add_pad: float = all_paddings - kernel_size
|
||||
one: float = 1.0
|
||||
for i in range(repeat_num):
|
||||
@@ -689,10 +690,12 @@ class ConvSubsampling(nn.Module):
|
||||
|
||||
class PositionalEncoding(torch.nn.Module):
|
||||
"""Fixed sinusoidal positional encoding.
|
||||
|
||||
Args:
|
||||
d_model (int): embedding dim
|
||||
max_len (int): maximum input length
|
||||
xscale (bool): whether to scale the input by sqrt(d_model)
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -724,12 +727,14 @@ class PositionalEncoding(torch.nn.Module):
|
||||
self, x: torch.Tensor, cache_len: int = 0
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Adds positional encoding.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): Input. Its shape is (batch, time, feature_size)
|
||||
cache_len (int): the size of the cache which is used to shift positions
|
||||
Returns:
|
||||
x+pos_emb (torch.Tensor): Its shape is (batch, time, feature_size)
|
||||
pos_emb (torch.Tensor): Its shape is (1, time, feature_size)
|
||||
|
||||
"""
|
||||
input_len = x.size(1) + cache_len
|
||||
if self.xscale:
|
||||
@@ -762,14 +767,15 @@ class RelPositionalEncoding(PositionalEncoding):
|
||||
self, x: torch.Tensor, cache_len: int = 0
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Compute positional encoding.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): Input. Its shape is (batch, time, feature_size)
|
||||
cache_len (int): the size of the cache which is used to shift positions
|
||||
Returns:
|
||||
x (torch.Tensor): Its shape is (batch, time, feature_size)
|
||||
pos_emb (torch.Tensor): Its shape is (1, time, feature_size)
|
||||
"""
|
||||
|
||||
"""
|
||||
if self.xscale:
|
||||
x = x * self.xscale
|
||||
|
||||
@@ -783,15 +789,13 @@ class RelPositionalEncoding(PositionalEncoding):
|
||||
|
||||
|
||||
class Swish(nn.SiLU):
|
||||
"""
|
||||
Swish activation function introduced in 'https://arxiv.org/abs/1710.05941'
|
||||
"""Swish activation function introduced in 'https://arxiv.org/abs/1710.05941'
|
||||
Mathematically identical to SiLU. See note in nn.SiLU for references.
|
||||
"""
|
||||
|
||||
|
||||
class ConformerFeedForward(nn.Module):
|
||||
"""
|
||||
feed-forward module of Conformer model.
|
||||
"""feed-forward module of Conformer model.
|
||||
use_bias (bool): Apply bias to all Linear and Conv1d
|
||||
layers to improve activation flow and stabilize
|
||||
training of huge models.
|
||||
@@ -819,8 +823,7 @@ class ConformerFeedForward(nn.Module):
|
||||
|
||||
|
||||
class CausalConv1D(nn.Conv1d):
|
||||
"""
|
||||
A causal version of nn.Conv1d where each step would
|
||||
"""A causal version of nn.Conv1d where each step would
|
||||
have limited access to locations on its right or left.
|
||||
All arguments are the same as nn.Conv1d except padding.
|
||||
|
||||
@@ -893,6 +896,7 @@ class CausalConv1D(nn.Conv1d):
|
||||
|
||||
class ConformerConvolution(nn.Module):
|
||||
"""The convolution module for the Conformer model.
|
||||
|
||||
Args:
|
||||
d_model (int): hidden dimension
|
||||
kernel_size (int): kernel size for depthwise convolution
|
||||
@@ -904,6 +908,7 @@ class ConformerConvolution(nn.Module):
|
||||
use_bias (bool): Use bias in all Linear and Conv1d
|
||||
layers to improve activation flow and stabilize
|
||||
training of huge models. Defaults to True
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -979,10 +984,12 @@ class ConformerConvolution(nn.Module):
|
||||
|
||||
class CohereASRMultiHeadAttention(nn.Module):
|
||||
"""Multi-Head Attention layer of Transformer.
|
||||
|
||||
Args:
|
||||
n_head (int): number of heads
|
||||
n_feat (int): size of the features
|
||||
use_bias (bool): whether to remove bias in linear and conv layers
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1010,6 +1017,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
value: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""Transforms query, key and value.
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): (batch, time1, size)
|
||||
key (torch.Tensor): (batch, time2, size)
|
||||
@@ -1018,6 +1026,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
q (torch.Tensor): (batch, head, time1, size)
|
||||
k (torch.Tensor): (batch, head, time2, size)
|
||||
v (torch.Tensor): (batch, head, time2, size)
|
||||
|
||||
"""
|
||||
n_batch = query.size(0)
|
||||
q = self.linear_q(query).view(n_batch, -1, self.h, self.d_k)
|
||||
@@ -1036,6 +1045,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
mask: torch.Tensor | None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute attention context vector.
|
||||
|
||||
Args:
|
||||
value (torch.Tensor): (batch, time2, size)
|
||||
scores(torch.Tensor): (batch, time1, time2)
|
||||
@@ -1044,6 +1054,7 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
value (torch.Tensor): transformed `value`
|
||||
(batch, time2, d_model) weighted by the
|
||||
attention scores
|
||||
|
||||
"""
|
||||
n_batch = value.size(0)
|
||||
if mask is not None:
|
||||
@@ -1071,16 +1082,19 @@ class CohereASRMultiHeadAttention(nn.Module):
|
||||
pos_emb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute 'Scaled Dot Product Attention'.
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): (batch, time1, size)
|
||||
key (torch.Tensor): (batch, time2, size)
|
||||
value(torch.Tensor): (batch, time2, size)
|
||||
mask (torch.Tensor): (batch, time1, time2)
|
||||
pos_emb (torch.Tensor): optional relative position embeddings
|
||||
|
||||
returns:
|
||||
Returns:
|
||||
output (torch.Tensor): transformed `value`
|
||||
(batch, time1, d_model) weighted by the
|
||||
query dot key attention
|
||||
|
||||
"""
|
||||
q, k, v = self.forward_qkv(query, key, value)
|
||||
|
||||
@@ -1130,8 +1144,10 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
|
||||
|
||||
def rel_shift(self, x: torch.Tensor) -> torch.Tensor:
|
||||
"""Compute relative positional encoding.
|
||||
|
||||
Args:
|
||||
x (torch.Tensor): (batch, nheads, time, 2*time-1)
|
||||
|
||||
"""
|
||||
b, h, qlen, pos_len = x.size() # (b, h, t1, t2)
|
||||
# need to add a column of zeros on the left side of
|
||||
@@ -1151,17 +1167,19 @@ class RelPositionMultiHeadAttention(CohereASRMultiHeadAttention):
|
||||
pos_emb: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""Compute 'Scaled Dot Product Attention' with rel. positional encoding.
|
||||
|
||||
Args:
|
||||
query (torch.Tensor): (batch, time1, size)
|
||||
key (torch.Tensor): (batch, time2, size)
|
||||
value(torch.Tensor): (batch, time2, size)
|
||||
mask (torch.Tensor): (batch, time1, time2)
|
||||
pos_emb (torch.Tensor) : (batch, time1, size)
|
||||
pos_emb (torch.Tensor): (batch, time1, size)
|
||||
|
||||
Returns:
|
||||
output (torch.Tensor): transformed `value`
|
||||
(batch, time1, d_model) weighted by the
|
||||
query dot key attention
|
||||
|
||||
"""
|
||||
q, k, v = self.forward_qkv(query, key, value)
|
||||
q = q.transpose(1, 2) # (batch, time1, head, d_k)
|
||||
@@ -1212,6 +1230,7 @@ class ConformerLayer(torch.nn.Module):
|
||||
Conv1d layers from each ConformerLayer to
|
||||
improve activation flow and stabilize training
|
||||
of huge models. Defaults to True.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
@@ -1279,14 +1298,14 @@ class ConformerLayer(torch.nn.Module):
|
||||
pos_emb: torch.Tensor | None = None,
|
||||
pad_mask: torch.Tensor | None = None,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
"""Args:
|
||||
x (torch.Tensor): input signals (B, T, d_model)
|
||||
att_mask (torch.Tensor): attention masks(B, T, T)
|
||||
pos_emb (torch.Tensor): (L, 1, d_model)
|
||||
pad_mask (torch.tensor): padding mask
|
||||
Returns:
|
||||
x (torch.Tensor): (B, T, d_model)
|
||||
|
||||
"""
|
||||
residual = x
|
||||
x = self.norm_feed_forward1(x)
|
||||
@@ -1331,8 +1350,7 @@ class ConformerLayer(torch.nn.Module):
|
||||
|
||||
|
||||
class ConformerEncoder(nn.Module):
|
||||
"""
|
||||
The encoder for ASR model of Conformer.
|
||||
"""The encoder for ASR model of Conformer.
|
||||
Based on this paper:
|
||||
'Conformer: Convolution-augmented Transformer for
|
||||
Speech Recognition' by Anmol Gulati et al.
|
||||
@@ -1473,12 +1491,12 @@ class ConformerEncoder(nn.Module):
|
||||
return num_encoder_cross_attn_tokens
|
||||
|
||||
def set_max_audio_length(self, max_audio_length: int) -> None:
|
||||
"""
|
||||
Sets maximum input length.
|
||||
"""Sets maximum input length.
|
||||
Pre-calculates internal seq_range mask.
|
||||
|
||||
Args:
|
||||
max_audio_length (int): New maximum sequence length.
|
||||
|
||||
"""
|
||||
device = next(self.parameters()).device
|
||||
dtype = next(self.parameters()).dtype
|
||||
|
||||
Reference in New Issue
Block a user