Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -138,14 +138,14 @@ class ApplyRotaryEmb(CustomOp):
|
||||
is_neox_style: bool = True,
|
||||
enable_fp32_compute: bool = False,
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Args:
|
||||
x: [batch_size (optional), seq_len, num_heads, head_size]
|
||||
cos: [seq_len, head_size // 2]
|
||||
sin: [seq_len, head_size // 2]
|
||||
is_neox_style: Whether to use the Neox-style or GPT-J-style.
|
||||
enable_fp32_compute: Temporarily convert x, cos, sin to FP32 dtype
|
||||
for higher accuracy.
|
||||
"""Args:
|
||||
x: [batch_size (optional), seq_len, num_heads, head_size]
|
||||
cos: [seq_len, head_size // 2]
|
||||
sin: [seq_len, head_size // 2]
|
||||
is_neox_style: Whether to use the Neox-style or GPT-J-style.
|
||||
enable_fp32_compute: Temporarily convert x, cos, sin to FP32 dtype
|
||||
for higher accuracy.
|
||||
|
||||
"""
|
||||
origin_dtype = x.dtype
|
||||
if enable_fp32_compute:
|
||||
|
||||
@@ -263,7 +263,6 @@ class DeepseekV4ScalingRotaryEmbedding(DeepseekScalingRotaryEmbedding):
|
||||
inverse: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""PyTorch-native implementation equivalent to forward()."""
|
||||
|
||||
head_size = query.size(-1)
|
||||
query_rot = query[..., -self.rotary_dim :]
|
||||
key_rot = key[..., -self.rotary_dim :] if key is not None else None
|
||||
|
||||
@@ -8,7 +8,7 @@ from .mrope import MRotaryEmbedding
|
||||
|
||||
|
||||
class Ernie4_5_VLRotaryEmbedding(MRotaryEmbedding):
|
||||
"""3D rotary positional embedding. 3D is t:time h:height w:width"""
|
||||
"""3D rotary positional embedding. 3D is t:time h:height w:width."""
|
||||
|
||||
def forward_native( # type: ignore[override]
|
||||
self,
|
||||
|
||||
@@ -185,7 +185,7 @@ class FourierRotaryEmbedding(RotaryEmbedding):
|
||||
return query, key
|
||||
|
||||
def weight_loader(self, param: nn.Parameter, loaded_weight: torch.Tensor):
|
||||
"""load fope weights"""
|
||||
"""Load fope weights."""
|
||||
world_size = get_tensor_model_parallel_world_size()
|
||||
rank = get_tensor_model_parallel_rank()
|
||||
num_key_value_heads = loaded_weight.size(0)
|
||||
|
||||
@@ -178,8 +178,12 @@ def triton_mrope(
|
||||
(T/H/W positions with multimodal inputs)
|
||||
mrope_section: [t, h, w]
|
||||
head_size: int
|
||||
rotary_dim: Number of leading dimensions rotary is applied to.
|
||||
mrope_interleaved: Whether the T/H/W sections are interleaved rather
|
||||
than concatenated.
|
||||
is_neox_style: Whether rotary pairs use split-half (NeoX) or
|
||||
adjacent (GPT-J) layout.
|
||||
|
||||
"""
|
||||
n_row, n_q_head_head_dim = q.shape
|
||||
n_q_head = n_q_head_head_dim // head_size
|
||||
@@ -320,6 +324,8 @@ class MRotaryEmbedding(RotaryEmbeddingBase):
|
||||
[3, num_tokens] (T/H/W positions with multimodal inputs)
|
||||
query: [num_tokens, num_heads * head_size]
|
||||
key: [num_tokens, num_kv_heads * head_size]
|
||||
offsets: Optional per-token position offsets added to positions.
|
||||
|
||||
"""
|
||||
assert positions.ndim == 1 or positions.ndim == 2
|
||||
assert key is not None
|
||||
|
||||
@@ -85,7 +85,7 @@ class MRotaryEmbeddingInterleaved(MRotaryEmbedding):
|
||||
self,
|
||||
positions: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor]:
|
||||
"""Interleave the rotary embedding"""
|
||||
"""Interleave the rotary embedding."""
|
||||
cos_sin = self.cos_sin_cache[positions]
|
||||
mrope_section_3d = [1] * len(self.mrope_dim)
|
||||
mrope_dim = self.mrope_dim
|
||||
@@ -139,8 +139,7 @@ class MRotaryEmbeddingInterleaved(MRotaryEmbedding):
|
||||
def get_mrope_interleaved_id_list(
|
||||
a: int, b: int, c: int, force_last: bool = False
|
||||
) -> list[int]:
|
||||
"""
|
||||
Generate an interleaved list of indices for multi-modal rotary embedding.
|
||||
"""Generate an interleaved list of indices for multi-modal rotary embedding.
|
||||
|
||||
Args:
|
||||
a: Number of indices for first modality
|
||||
@@ -150,6 +149,7 @@ class MRotaryEmbeddingInterleaved(MRotaryEmbedding):
|
||||
|
||||
Returns:
|
||||
List of interleaved indices
|
||||
|
||||
"""
|
||||
if force_last:
|
||||
a -= 1
|
||||
|
||||
@@ -9,7 +9,8 @@ from .base import RotaryEmbedding
|
||||
|
||||
class NTKScalingRotaryEmbedding(RotaryEmbedding):
|
||||
"""RotaryEmbedding extended with fixed and mixed NTK scaling.
|
||||
https://kexue.fm/archives/9706"""
|
||||
https://kexue.fm/archives/9706
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user