Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -244,8 +244,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
def _apply_async_impl(
|
||||
self, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass with base linear and LoRA on separate CUDA streams
|
||||
"""Forward pass with base linear and LoRA on separate CUDA streams
|
||||
for overlap, using maybe_execute_in_parallel.
|
||||
Base layer runs on default stream; LoRA runs on aux stream.
|
||||
"""
|
||||
|
||||
@@ -83,8 +83,7 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
|
||||
|
||||
class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
"""
|
||||
LoRA on top of ColumnParallelLinear layer.
|
||||
"""LoRA on top of ColumnParallelLinear layer.
|
||||
LoRA B is sliced for tensor parallelism.
|
||||
There are two types for the `base_layer`:
|
||||
1. ColumnParallelLinear, e.g.`dense_h_to_4h` in `FalconForCausalLM`.
|
||||
@@ -140,6 +139,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
Returns:
|
||||
- output
|
||||
- bias
|
||||
|
||||
"""
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
|
||||
@@ -210,8 +210,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
The main reason for overriding this function is to enhance code
|
||||
"""The main reason for overriding this function is to enhance code
|
||||
maintainability.
|
||||
"""
|
||||
self.lora_config = lora_config
|
||||
@@ -268,8 +267,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
lora_a: list[torch.Tensor | None],
|
||||
lora_b: list[torch.Tensor | None],
|
||||
) -> tuple[list[torch.Tensor | None], list[torch.Tensor | None]]:
|
||||
"""
|
||||
Expand packed adapter groups when they don't match n_slices.
|
||||
"""Expand packed adapter groups when they don't match n_slices.
|
||||
E.g. in_proj_qkv (covers Q+K+V) + in_proj_z.
|
||||
|
||||
A None group member means that member was not adapted; the slice(s)
|
||||
@@ -388,8 +386,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
ColumnParallelLinear layer that is specifically designed for
|
||||
"""ColumnParallelLinear layer that is specifically designed for
|
||||
qkv_proj. Certain models, such as chatglm3 and baichuan-7b,
|
||||
only contains a single LoRA within their qkv_proj layer.
|
||||
|
||||
@@ -493,8 +490,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
The main reason for overloading this function is to handle inconsistent
|
||||
"""The main reason for overloading this function is to handle inconsistent
|
||||
weight dimensions in qkv lora.
|
||||
"""
|
||||
super().create_lora_weights(max_loras, lora_config, model_config)
|
||||
@@ -521,8 +517,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
|
||||
"""Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
"""
|
||||
@@ -562,8 +557,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from MergedColumnParallelLinearWithLoRA by slicing the
|
||||
"""Differs from MergedColumnParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -605,8 +599,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
|
||||
|
||||
|
||||
class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from QKVParallelLinearWithLoRA by slicing the
|
||||
"""Differs from QKVParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -641,8 +634,7 @@ class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from MergedQKVParallelLinearWithLoRA by slicing the
|
||||
"""Differs from MergedQKVParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -741,7 +733,8 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
|
||||
lora_b: torch.Tensor | list[torch.Tensor],
|
||||
):
|
||||
"""Override to handle single tensor weights
|
||||
that need to be split into slices."""
|
||||
that need to be split into slices.
|
||||
"""
|
||||
self.reset_lora(index)
|
||||
|
||||
# Handle case where checkpoint has single tensor weights
|
||||
|
||||
@@ -248,7 +248,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""Initializes lora matrices."""
|
||||
|
||||
self._verify_ep_fs(lora_config)
|
||||
self.max_loras = lora_config.max_loras
|
||||
self.fully_sharded = lora_config.fully_sharded_loras
|
||||
@@ -294,9 +293,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
)
|
||||
|
||||
def _slice_w13_a(self, w13_lora_a: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
|
||||
"""
|
||||
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
|
||||
if self.tp_size == 1 or not self.fully_sharded:
|
||||
return w13_lora_a
|
||||
|
||||
@@ -321,9 +318,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
return w13_lora_b[:, start_idx:end_idx, :]
|
||||
|
||||
def _slice_w2_a(self, w2_lora_a: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
|
||||
"""
|
||||
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
|
||||
if self.tp_size == 1:
|
||||
return w2_lora_a
|
||||
# w2_lora_a shape (num_experts,rank,input_size)
|
||||
@@ -334,9 +329,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
return w2_lora_a[:, :, start_idx:end_idx]
|
||||
|
||||
def _slice_w2_b(self, w2_lora_b: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
|
||||
"""
|
||||
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
|
||||
if self.tp_size == 1 or not self.fully_sharded:
|
||||
return w2_lora_b
|
||||
# Based on S-LoRA, we slice W2 B along the hidden_size dim.
|
||||
@@ -475,7 +468,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> bool:
|
||||
"""Returns True if the layer can be replaced by this LoRA layer."""
|
||||
|
||||
# source_layer is MoERunner
|
||||
moe_cls = maybe_get_oot_by_class(MoERunner)
|
||||
return isinstance(source_layer, moe_cls) and len(packed_modules_list) == 2
|
||||
@@ -522,7 +514,6 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""Initializes lora matrices."""
|
||||
|
||||
if model_config is None:
|
||||
raise ValueError("model_config must be provided for MoE LoRA.")
|
||||
architectures = model_config.architectures
|
||||
@@ -609,30 +600,22 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
|
||||
|
||||
@property
|
||||
def w13_input_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.w13_lora_a_stacked[0].shape[-1]
|
||||
|
||||
@property
|
||||
def w13_output_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.w13_lora_b_stacked[0].shape[-2] * self.tp_size
|
||||
|
||||
@property
|
||||
def w2_input_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.w2_lora_a_stacked[0].shape[-1] * self.tp_size
|
||||
|
||||
@property
|
||||
def w2_output_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.hidden_size
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -18,8 +18,7 @@ from .base import BaseLayerWithLoRA
|
||||
|
||||
|
||||
class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
"""
|
||||
LoRA wrapper for LogitsProcessor, with extra logic to handle the
|
||||
"""LoRA wrapper for LogitsProcessor, with extra logic to handle the
|
||||
application of the LoRA adapter.
|
||||
|
||||
Args:
|
||||
@@ -30,6 +29,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
sharded_to_full_mapping: index mapping from sharded vocab to full vocab
|
||||
received from base_layer.get_sharded_to_full_mapping(). If None,
|
||||
no reindexing will be done.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -25,7 +25,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
def forward(
|
||||
self, input_: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Forward of ReplicatedLinearWithLoRA
|
||||
"""Forward of ReplicatedLinearWithLoRA.
|
||||
|
||||
Args:
|
||||
input_: Tensor whose last dimension is `input_size`.
|
||||
@@ -33,6 +33,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
Returns:
|
||||
- output
|
||||
- bias
|
||||
|
||||
"""
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
def forward(
|
||||
self, input_: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Forward of RowParallelLinear
|
||||
"""Forward of RowParallelLinear.
|
||||
|
||||
Args:
|
||||
input_: tensor whose last dimension is `input_size`. If
|
||||
@@ -52,6 +52,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
Returns:
|
||||
- output
|
||||
- bias
|
||||
|
||||
"""
|
||||
# set up backprop all-reduce.
|
||||
if self.base_layer.input_is_parallel:
|
||||
@@ -99,8 +100,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
|
||||
|
||||
class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from RowParallelLinearWithLoRA by slicing the
|
||||
"""Differs from RowParallelLinearWithLoRA by slicing the
|
||||
LoRA B's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the output dim.
|
||||
|
||||
@@ -74,8 +74,7 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
|
||||
|
||||
|
||||
def _not_fully_sharded_can_replace(can_replace):
|
||||
"""
|
||||
decorator which adds the condition of not using fully sharded loras
|
||||
"""Decorator which adds the condition of not using fully sharded loras
|
||||
intended to wrap can_replace_layer()
|
||||
"""
|
||||
|
||||
@@ -88,8 +87,7 @@ def _not_fully_sharded_can_replace(can_replace):
|
||||
|
||||
|
||||
def _fully_sharded_can_replace(can_replace):
|
||||
"""
|
||||
decorator which adds the condition of fully sharded loras
|
||||
"""Decorator which adds the condition of fully sharded loras
|
||||
intended to wrap can_replace_layer()
|
||||
"""
|
||||
|
||||
|
||||
Reference in New Issue
Block a user