Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -2,18 +2,23 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
from vllm.lora.layers.base import BaseLayerWithLoRA
|
||||
from vllm.lora.layers.column_parallel_linear import (
|
||||
ColumnParallelLinearWithLoRA, ColumnParallelLinearWithShardedLoRA,
|
||||
ColumnParallelLinearWithLoRA,
|
||||
ColumnParallelLinearWithShardedLoRA,
|
||||
MergedColumnParallelLinearWithLoRA,
|
||||
MergedColumnParallelLinearWithShardedLoRA, MergedQKVParallelLinearWithLoRA,
|
||||
MergedQKVParallelLinearWithShardedLoRA, QKVParallelLinearWithLoRA,
|
||||
QKVParallelLinearWithShardedLoRA)
|
||||
MergedColumnParallelLinearWithShardedLoRA,
|
||||
MergedQKVParallelLinearWithLoRA,
|
||||
MergedQKVParallelLinearWithShardedLoRA,
|
||||
QKVParallelLinearWithLoRA,
|
||||
QKVParallelLinearWithShardedLoRA,
|
||||
)
|
||||
from vllm.lora.layers.logits_processor import LogitsProcessorWithLoRA
|
||||
from vllm.lora.layers.replicated_linear import ReplicatedLinearWithLoRA
|
||||
from vllm.lora.layers.row_parallel_linear import (
|
||||
RowParallelLinearWithLoRA, RowParallelLinearWithShardedLoRA)
|
||||
RowParallelLinearWithLoRA,
|
||||
RowParallelLinearWithShardedLoRA,
|
||||
)
|
||||
from vllm.lora.layers.utils import LoRAMapping
|
||||
from vllm.lora.layers.vocal_parallel_embedding import (
|
||||
VocabParallelEmbeddingWithLoRA)
|
||||
from vllm.lora.layers.vocal_parallel_embedding import VocabParallelEmbeddingWithLoRA
|
||||
|
||||
__all__ = [
|
||||
"BaseLayerWithLoRA",
|
||||
|
||||
@@ -14,7 +14,6 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class BaseLayerWithLoRA(nn.Module):
|
||||
|
||||
def slice_lora_a(
|
||||
self, lora_a: Union[torch.Tensor, list[Union[torch.Tensor, None]]]
|
||||
) -> Union[torch.Tensor, list[Union[torch.Tensor, None]]]:
|
||||
|
||||
@@ -8,10 +8,14 @@ from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.distributed.utils import divide
|
||||
|
||||
# yapf: disable
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
LinearBase, ReplicatedLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
LinearBase,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import BaseLayerWithLoRA
|
||||
|
||||
@@ -10,9 +10,11 @@ from transformers import PretrainedConfig
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.distributed import tensor_model_parallel_all_gather
|
||||
from vllm.distributed.utils import divide
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear)
|
||||
from vllm.model_executor.layers.linear import (
|
||||
ColumnParallelLinear,
|
||||
MergedColumnParallelLinear,
|
||||
QKVParallelLinear,
|
||||
)
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base_linear import BaseLinearLayerWithLoRA
|
||||
@@ -20,12 +22,16 @@ from .utils import _fully_sharded_can_replace, _not_fully_sharded_can_replace
|
||||
|
||||
|
||||
def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
"""
|
||||
For `ColumnParallelLinearWithLoRA` or classes that inherit from
|
||||
"""
|
||||
For `ColumnParallelLinearWithLoRA` or classes that inherit from
|
||||
`ColumnParallelLinearWithLoRA`, they share the same `apply` logic.
|
||||
"""
|
||||
assert (layer.n_slices == len(layer.lora_a_stacked) == len(
|
||||
layer.lora_b_stacked) == len(layer.output_slices))
|
||||
assert (
|
||||
layer.n_slices
|
||||
== len(layer.lora_a_stacked)
|
||||
== len(layer.lora_b_stacked)
|
||||
== len(layer.output_slices)
|
||||
)
|
||||
if layer.lora_bias_stacked is not None:
|
||||
assert layer.n_slices == len(layer.lora_bias_stacked)
|
||||
|
||||
@@ -43,7 +49,8 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
)
|
||||
|
||||
shrunk_buffers: Optional[torch.Tensor] = layer.punica_wrapper.add_shrink(
|
||||
buffers, x, layer.lora_a_stacked, 1.0)
|
||||
buffers, x, layer.lora_a_stacked, 1.0
|
||||
)
|
||||
|
||||
if not current_platform.can_update_inplace():
|
||||
buffers = shrunk_buffers
|
||||
@@ -57,7 +64,8 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
layer.lora_bias_stacked,
|
||||
layer.output_slices,
|
||||
offset_start=0,
|
||||
add_input=True)
|
||||
add_input=True,
|
||||
)
|
||||
|
||||
if not current_platform.can_update_inplace():
|
||||
output = lora_output
|
||||
@@ -81,8 +89,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
# The base_layer type is ColumnParallelLinear or
|
||||
# MergedColumnParallelLinear, their weight sharding logic is
|
||||
# inconsistent when TP is greater than 1.
|
||||
self.is_merged_col_linear = type(
|
||||
base_layer) is MergedColumnParallelLinear
|
||||
self.is_merged_col_linear = type(base_layer) is MergedColumnParallelLinear
|
||||
self.output_size = self.base_layer.output_size_per_partition
|
||||
# There is only one LoRA layer
|
||||
self.n_slices = 1
|
||||
@@ -97,10 +104,14 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
shard_size = self.output_size // 2
|
||||
offset = lora_b.shape[0] // 2
|
||||
|
||||
left_weight = lora_b[self.tp_rank * shard_size:(self.tp_rank + 1) *
|
||||
shard_size, :]
|
||||
right_weight = lora_b[offset + self.tp_rank * shard_size:offset +
|
||||
(self.tp_rank + 1) * shard_size, :]
|
||||
left_weight = lora_b[
|
||||
self.tp_rank * shard_size : (self.tp_rank + 1) * shard_size, :
|
||||
]
|
||||
right_weight = lora_b[
|
||||
offset + self.tp_rank * shard_size : offset
|
||||
+ (self.tp_rank + 1) * shard_size,
|
||||
:,
|
||||
]
|
||||
lora_b = torch.cat([left_weight, right_weight], dim=0)
|
||||
# Applicable to cases where the base_layer is
|
||||
# ColumnParallelLinear.
|
||||
@@ -133,8 +144,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
- output
|
||||
- bias
|
||||
"""
|
||||
bias = (self.base_layer.bias
|
||||
if not self.base_layer.skip_bias_add else None)
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
|
||||
# Matrix multiply.
|
||||
output_parallel = self.apply(input_, bias)
|
||||
@@ -147,8 +157,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
if not self.base_layer.return_bias:
|
||||
return output
|
||||
|
||||
output_bias = (self.base_layer.bias
|
||||
if self.base_layer.skip_bias_add else None)
|
||||
output_bias = self.base_layer.bias if self.base_layer.skip_bias_add else None
|
||||
return output, output_bias
|
||||
|
||||
@classmethod
|
||||
@@ -162,7 +171,8 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
) -> bool:
|
||||
return type(source_layer) is ColumnParallelLinear or (
|
||||
type(source_layer) is MergedColumnParallelLinear
|
||||
and len(packed_modules_list) == 1)
|
||||
and len(packed_modules_list) == 1
|
||||
)
|
||||
|
||||
|
||||
class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
@@ -175,17 +185,18 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self, base_layer: Union[MergedColumnParallelLinear,
|
||||
QKVParallelLinear]) -> None:
|
||||
self, base_layer: Union[MergedColumnParallelLinear, QKVParallelLinear]
|
||||
) -> None:
|
||||
super().__init__(base_layer)
|
||||
# There are two LoRA layers
|
||||
# the output_sizes in MergedColumnParallelLinear is not sharded by tp
|
||||
# we need to divide it by the tp_size to get correct slices size
|
||||
output_sizes = self.base_layer.output_sizes
|
||||
self.output_slices = tuple(
|
||||
divide(output_size, self.tp_size) for output_size in output_sizes)
|
||||
divide(output_size, self.tp_size) for output_size in output_sizes
|
||||
)
|
||||
self.n_slices = len(self.output_slices)
|
||||
self.output_ids = (self.tp_rank, ) * self.n_slices
|
||||
self.output_ids = (self.tp_rank,) * self.n_slices
|
||||
|
||||
def create_lora_weights(
|
||||
self,
|
||||
@@ -194,14 +205,16 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
model_config: Optional[PretrainedConfig] = None,
|
||||
) -> None:
|
||||
"""
|
||||
The main reason for overriding this function is to enhance code
|
||||
The main reason for overriding this function is to enhance code
|
||||
maintainability.
|
||||
"""
|
||||
self.lora_config = lora_config
|
||||
|
||||
lora_a_output_size_per_partition = (
|
||||
lora_config.max_lora_rank if not lora_config.fully_sharded_loras
|
||||
else divide(lora_config.max_lora_rank, self.tp_size))
|
||||
lora_config.max_lora_rank
|
||||
if not lora_config.fully_sharded_loras
|
||||
else divide(lora_config.max_lora_rank, self.tp_size)
|
||||
)
|
||||
|
||||
self.lora_a_stacked = tuple(
|
||||
torch.zeros(
|
||||
@@ -211,7 +224,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
self.input_size,
|
||||
dtype=lora_config.lora_dtype,
|
||||
device=self.device,
|
||||
) for _ in range(self.n_slices))
|
||||
)
|
||||
for _ in range(self.n_slices)
|
||||
)
|
||||
self.lora_b_stacked = tuple(
|
||||
torch.zeros(
|
||||
max_loras,
|
||||
@@ -220,7 +235,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
lora_config.max_lora_rank,
|
||||
dtype=lora_config.lora_dtype,
|
||||
device=self.device,
|
||||
) for output_size in self.output_slices)
|
||||
)
|
||||
for output_size in self.output_slices
|
||||
)
|
||||
if lora_config.bias_enabled:
|
||||
self.lora_bias_stacked = tuple(
|
||||
torch.zeros(
|
||||
@@ -229,7 +246,9 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
output_size,
|
||||
dtype=lora_config.lora_dtype,
|
||||
device=self.device,
|
||||
) for output_size in self.output_slices)
|
||||
)
|
||||
for output_size in self.output_slices
|
||||
)
|
||||
|
||||
def slice_lora_a(
|
||||
self, lora_a: list[Union[torch.Tensor, None]]
|
||||
@@ -241,20 +260,22 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
) -> list[Union[torch.Tensor, None]]:
|
||||
sliced_lora_b = [None] * self.n_slices
|
||||
for i, (shard_id, shard_size) in enumerate(
|
||||
zip(self.output_ids, self.output_slices)):
|
||||
zip(self.output_ids, self.output_slices)
|
||||
):
|
||||
if (lora_b_i := lora_b[i]) is not None:
|
||||
sliced_lora_b[i] = lora_b_i[shard_size * shard_id:shard_size *
|
||||
(shard_id + 1), :]
|
||||
sliced_lora_b[i] = lora_b_i[
|
||||
shard_size * shard_id : shard_size * (shard_id + 1), :
|
||||
]
|
||||
return sliced_lora_b
|
||||
|
||||
def slice_bias(
|
||||
self, bias: list[Union[torch.Tensor,
|
||||
None]]) -> list[Union[torch.Tensor, None]]:
|
||||
self, bias: list[Union[torch.Tensor, None]]
|
||||
) -> list[Union[torch.Tensor, None]]:
|
||||
for i, (shard_id, shard_size) in enumerate(
|
||||
zip(self.output_ids, self.output_slices)):
|
||||
zip(self.output_ids, self.output_slices)
|
||||
):
|
||||
if (bias_i := bias[i]) is not None:
|
||||
bias[i] = bias_i[shard_size * shard_id:shard_size *
|
||||
(shard_id + 1)]
|
||||
bias[i] = bias_i[shard_size * shard_id : shard_size * (shard_id + 1)]
|
||||
return bias
|
||||
|
||||
def set_lora(
|
||||
@@ -276,22 +297,22 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
for i in range(self.n_slices):
|
||||
if (lora_a_i := lora_a[i]) is not None:
|
||||
self.lora_a_stacked[i][
|
||||
index, 0, :lora_a_i.shape[0], :lora_a_i.shape[1]].copy_(
|
||||
lora_a_i, non_blocking=True)
|
||||
index, 0, : lora_a_i.shape[0], : lora_a_i.shape[1]
|
||||
].copy_(lora_a_i, non_blocking=True)
|
||||
if (lora_b_i := lora_b[i]) is not None:
|
||||
self.lora_b_stacked[i][
|
||||
index, 0, :lora_b_i.shape[0], :lora_b_i.shape[1]].copy_(
|
||||
lora_b_i, non_blocking=True)
|
||||
index, 0, : lora_b_i.shape[0], : lora_b_i.shape[1]
|
||||
].copy_(lora_b_i, non_blocking=True)
|
||||
|
||||
if lora_bias is not None:
|
||||
self.lora_bias_stacked = cast(tuple[torch.Tensor, ...],
|
||||
self.lora_bias_stacked)
|
||||
self.lora_bias_stacked = cast(
|
||||
tuple[torch.Tensor, ...], self.lora_bias_stacked
|
||||
)
|
||||
for i in range(self.n_slices):
|
||||
if (lora_bias_i := lora_bias[i]) is not None:
|
||||
self.lora_bias_stacked[i][index,
|
||||
0, :lora_bias_i.shape[0]].copy_(
|
||||
lora_bias_i,
|
||||
non_blocking=True)
|
||||
self.lora_bias_stacked[i][index, 0, : lora_bias_i.shape[0]].copy_(
|
||||
lora_bias_i, non_blocking=True
|
||||
)
|
||||
|
||||
@classmethod
|
||||
@_not_fully_sharded_can_replace
|
||||
@@ -302,8 +323,10 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
packed_modules_list: list,
|
||||
model_config: Optional[PretrainedConfig],
|
||||
) -> bool:
|
||||
return (type(source_layer) is MergedColumnParallelLinear
|
||||
and len(packed_modules_list) == 2)
|
||||
return (
|
||||
type(source_layer) is MergedColumnParallelLinear
|
||||
and len(packed_modules_list) == 2
|
||||
)
|
||||
|
||||
|
||||
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
@@ -321,57 +344,70 @@ class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
def __init__(self, base_layer: QKVParallelLinear) -> None:
|
||||
super().__init__(base_layer)
|
||||
self.q_proj_total_size = (self.base_layer.total_num_heads *
|
||||
self.base_layer.head_size)
|
||||
self.q_proj_shard_size = (self.base_layer.num_heads *
|
||||
self.base_layer.head_size)
|
||||
self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
|
||||
self.base_layer.head_size)
|
||||
self.kv_proj_total_size = (self.base_layer.total_num_kv_heads *
|
||||
self.base_layer.head_size)
|
||||
self.q_proj_total_size = (
|
||||
self.base_layer.total_num_heads * self.base_layer.head_size
|
||||
)
|
||||
self.q_proj_shard_size = self.base_layer.num_heads * self.base_layer.head_size
|
||||
self.kv_proj_shard_size = (
|
||||
self.base_layer.num_kv_heads * self.base_layer.head_size
|
||||
)
|
||||
self.kv_proj_total_size = (
|
||||
self.base_layer.total_num_kv_heads * self.base_layer.head_size
|
||||
)
|
||||
# There is only one LoRA layer
|
||||
self.n_slices = 1
|
||||
|
||||
def slice_lora_b(self, lora_b: torch.Tensor) -> torch.Tensor:
|
||||
|
||||
self.q_shard_id = self.tp_rank
|
||||
self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas
|
||||
lora_b_q = lora_b[self.q_proj_shard_size *
|
||||
self.q_shard_id:self.q_proj_shard_size *
|
||||
(self.q_shard_id + 1), :]
|
||||
lora_b_q = lora_b[
|
||||
self.q_proj_shard_size * self.q_shard_id : self.q_proj_shard_size
|
||||
* (self.q_shard_id + 1),
|
||||
:,
|
||||
]
|
||||
k_offset = self.q_proj_total_size
|
||||
lora_b_k = lora_b[k_offset +
|
||||
self.kv_proj_shard_size * self.kv_shard_id:k_offset +
|
||||
self.kv_proj_shard_size * (self.kv_shard_id + 1), :]
|
||||
lora_b_k = lora_b[
|
||||
k_offset + self.kv_proj_shard_size * self.kv_shard_id : k_offset
|
||||
+ self.kv_proj_shard_size * (self.kv_shard_id + 1),
|
||||
:,
|
||||
]
|
||||
v_offset = k_offset + self.kv_proj_total_size
|
||||
lora_b_v = lora_b[v_offset +
|
||||
self.kv_proj_shard_size * self.kv_shard_id:v_offset +
|
||||
self.kv_proj_shard_size * (self.kv_shard_id + 1), :]
|
||||
lora_b_v = lora_b[
|
||||
v_offset + self.kv_proj_shard_size * self.kv_shard_id : v_offset
|
||||
+ self.kv_proj_shard_size * (self.kv_shard_id + 1),
|
||||
:,
|
||||
]
|
||||
lora_b = torch.cat([lora_b_q, lora_b_k, lora_b_v], dim=0)
|
||||
return lora_b
|
||||
|
||||
def slice_bias(self, bias: torch.Tensor) -> torch.Tensor:
|
||||
bias_q = bias[self.q_proj_shard_size *
|
||||
self.q_shard_id:self.q_proj_shard_size *
|
||||
(self.q_shard_id + 1)]
|
||||
bias_q = bias[
|
||||
self.q_proj_shard_size * self.q_shard_id : self.q_proj_shard_size
|
||||
* (self.q_shard_id + 1)
|
||||
]
|
||||
k_offset = self.q_proj_total_size
|
||||
bias_k = bias[k_offset +
|
||||
self.kv_proj_shard_size * self.kv_shard_id:k_offset +
|
||||
self.kv_proj_shard_size * (self.kv_shard_id + 1)]
|
||||
bias_k = bias[
|
||||
k_offset + self.kv_proj_shard_size * self.kv_shard_id : k_offset
|
||||
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)
|
||||
]
|
||||
v_offset = k_offset + self.kv_proj_total_size
|
||||
bias_v = bias[v_offset +
|
||||
self.kv_proj_shard_size * self.kv_shard_id:v_offset +
|
||||
self.kv_proj_shard_size * (self.kv_shard_id + 1)]
|
||||
bias_v = bias[
|
||||
v_offset + self.kv_proj_shard_size * self.kv_shard_id : v_offset
|
||||
+ self.kv_proj_shard_size * (self.kv_shard_id + 1)
|
||||
]
|
||||
bias = torch.cat([bias_q, bias_k, bias_v], dim=1)
|
||||
return bias
|
||||
|
||||
@classmethod
|
||||
@_not_fully_sharded_can_replace
|
||||
def can_replace_layer(cls, source_layer: nn.Module,
|
||||
lora_config: LoRAConfig, packed_modules_list: list,
|
||||
model_config: Optional[PretrainedConfig]) -> bool:
|
||||
return type(source_layer) is QKVParallelLinear and len(
|
||||
packed_modules_list) == 1
|
||||
def can_replace_layer(
|
||||
cls,
|
||||
source_layer: nn.Module,
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: Optional[PretrainedConfig],
|
||||
) -> bool:
|
||||
return type(source_layer) is QKVParallelLinear and len(packed_modules_list) == 1
|
||||
|
||||
|
||||
class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
@@ -390,10 +426,10 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
# There are three LoRA layer.
|
||||
self.n_slices = len(self.base_layer.output_sizes)
|
||||
|
||||
self.q_proj_shard_size = (self.base_layer.num_heads *
|
||||
self.base_layer.head_size)
|
||||
self.kv_proj_shard_size = (self.base_layer.num_kv_heads *
|
||||
self.base_layer.head_size)
|
||||
self.q_proj_shard_size = self.base_layer.num_heads * self.base_layer.head_size
|
||||
self.kv_proj_shard_size = (
|
||||
self.base_layer.num_kv_heads * self.base_layer.head_size
|
||||
)
|
||||
self.q_shard_id = self.tp_rank
|
||||
self.kv_shard_id = self.tp_rank // self.base_layer.num_kv_head_replicas
|
||||
|
||||
@@ -415,7 +451,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
model_config: Optional[PretrainedConfig] = None,
|
||||
) -> None:
|
||||
"""
|
||||
The main reason for overloading this function is to handle inconsistent
|
||||
The main reason for overloading this function is to handle inconsistent
|
||||
weight dimensions in qkv lora.
|
||||
"""
|
||||
super().create_lora_weights(max_loras, lora_config, model_config)
|
||||
@@ -429,8 +465,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
packed_modules_list: list,
|
||||
model_config: Optional[PretrainedConfig],
|
||||
) -> bool:
|
||||
return (type(source_layer) is QKVParallelLinear
|
||||
and len(packed_modules_list) == 3)
|
||||
return type(source_layer) is QKVParallelLinear and len(packed_modules_list) == 3
|
||||
|
||||
|
||||
# These following layers are based on the tensor parallelism strategy given in
|
||||
@@ -453,12 +488,12 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
|
||||
shard_size = self.lora_a_stacked[0].shape[2]
|
||||
start_idx = self.tp_rank * shard_size
|
||||
lora_a = lora_a[start_idx:start_idx + shard_size, :]
|
||||
lora_a = lora_a[start_idx : start_idx + shard_size, :]
|
||||
return lora_a
|
||||
|
||||
def apply(self,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
def apply(
|
||||
self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
return _mcp_apply(x, bias, self)
|
||||
|
||||
@classmethod
|
||||
@@ -480,8 +515,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
)
|
||||
|
||||
|
||||
class MergedColumnParallelLinearWithShardedLoRA(
|
||||
MergedColumnParallelLinearWithLoRA):
|
||||
class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from MergedColumnParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
@@ -492,20 +526,22 @@ class MergedColumnParallelLinearWithShardedLoRA(
|
||||
def slice_lora_a(
|
||||
self, lora_a: list[Union[torch.Tensor, None]]
|
||||
) -> list[Union[torch.Tensor, None]]:
|
||||
#NOTE: lora_a contains 2 subloras, and each sublora could be None.
|
||||
# NOTE: lora_a contains 2 subloras, and each sublora could be None.
|
||||
output_shard_size = self.lora_a_stacked[0].shape[2]
|
||||
output_start_idx = self.tp_rank * output_shard_size
|
||||
lora_a = [
|
||||
lora_a[0][output_start_idx:output_start_idx +
|
||||
output_shard_size, :] if lora_a[0] is not None else None,
|
||||
lora_a[1][output_start_idx:output_start_idx +
|
||||
output_shard_size, :] if lora_a[1] is not None else None,
|
||||
lora_a[0][output_start_idx : output_start_idx + output_shard_size, :]
|
||||
if lora_a[0] is not None
|
||||
else None,
|
||||
lora_a[1][output_start_idx : output_start_idx + output_shard_size, :]
|
||||
if lora_a[1] is not None
|
||||
else None,
|
||||
]
|
||||
return lora_a
|
||||
|
||||
def apply(self,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
def apply(
|
||||
self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
return _mcp_apply(x, bias, self)
|
||||
|
||||
@classmethod
|
||||
@@ -538,19 +574,23 @@ class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
|
||||
def slice_lora_a(self, lora_a: torch.Tensor) -> torch.Tensor:
|
||||
shard_size = self.lora_a_stacked[0].shape[2]
|
||||
start_idx = self.tp_rank * shard_size
|
||||
lora_a = lora_a[start_idx:start_idx + shard_size, :]
|
||||
lora_a = lora_a[start_idx : start_idx + shard_size, :]
|
||||
return lora_a
|
||||
|
||||
def apply(self,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
def apply(
|
||||
self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
return _mcp_apply(x, bias, self)
|
||||
|
||||
@classmethod
|
||||
@_fully_sharded_can_replace
|
||||
def can_replace_layer(cls, source_layer: nn.Module,
|
||||
lora_config: LoRAConfig, packed_modules_list: list,
|
||||
model_config: Optional[PretrainedConfig]) -> bool:
|
||||
def can_replace_layer(
|
||||
cls,
|
||||
source_layer: nn.Module,
|
||||
lora_config: LoRAConfig,
|
||||
packed_modules_list: list,
|
||||
model_config: Optional[PretrainedConfig],
|
||||
) -> bool:
|
||||
# specifying kwargs so they can be easily accessed in decorator
|
||||
return super().can_replace_layer(
|
||||
source_layer=source_layer,
|
||||
@@ -563,7 +603,7 @@ class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
|
||||
|
||||
class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from MergedQKVParallelLinearWithLoRA by slicing the
|
||||
Differs from MergedQKVParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -576,18 +616,21 @@ class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
|
||||
shard_size = [self.lora_a_stacked[i].shape[2] for i in range(3)]
|
||||
start_idx = [self.tp_rank * shard_size[i] for i in range(3)]
|
||||
lora_a = [
|
||||
lora_a[0][start_idx[0]:start_idx[0] +
|
||||
shard_size[0], :] if lora_a[0] is not None else None,
|
||||
lora_a[1][start_idx[1]:start_idx[1] +
|
||||
shard_size[1], :] if lora_a[1] is not None else None,
|
||||
lora_a[2][start_idx[2]:start_idx[2] +
|
||||
shard_size[2], :] if lora_a[2] is not None else None,
|
||||
lora_a[0][start_idx[0] : start_idx[0] + shard_size[0], :]
|
||||
if lora_a[0] is not None
|
||||
else None,
|
||||
lora_a[1][start_idx[1] : start_idx[1] + shard_size[1], :]
|
||||
if lora_a[1] is not None
|
||||
else None,
|
||||
lora_a[2][start_idx[2] : start_idx[2] + shard_size[2], :]
|
||||
if lora_a[2] is not None
|
||||
else None,
|
||||
]
|
||||
return lora_a
|
||||
|
||||
def apply(self,
|
||||
x: torch.Tensor,
|
||||
bias: Optional[torch.Tensor] = None) -> torch.Tensor:
|
||||
def apply(
|
||||
self, x: torch.Tensor, bias: Optional[torch.Tensor] = None
|
||||
) -> torch.Tensor:
|
||||
return _mcp_apply(x, bias, self)
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -9,11 +9,12 @@ import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.distributed import (get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size)
|
||||
from vllm.distributed import (
|
||||
get_tensor_model_parallel_rank,
|
||||
get_tensor_model_parallel_world_size,
|
||||
)
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import BaseLayerWithLoRA
|
||||
@@ -34,9 +35,14 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
no reindexing will be done.
|
||||
"""
|
||||
|
||||
def __init__(self, base_layer: LogitsProcessor, hidden_size: int,
|
||||
dtype: torch.dtype, device: torch.device,
|
||||
sharded_to_full_mapping: Optional[list[int]]) -> None:
|
||||
def __init__(
|
||||
self,
|
||||
base_layer: LogitsProcessor,
|
||||
hidden_size: int,
|
||||
dtype: torch.dtype,
|
||||
device: torch.device,
|
||||
sharded_to_full_mapping: Optional[list[int]],
|
||||
) -> None:
|
||||
super().__init__()
|
||||
self.base_layer = base_layer
|
||||
self.hidden_size = hidden_size
|
||||
@@ -86,8 +92,9 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
) -> None:
|
||||
# TODO: Verify if this condition can be further relaxed
|
||||
if 32000 < self.base_layer.vocab_size > 257024:
|
||||
raise ValueError("When using LoRA, vocab size must be "
|
||||
"32000 >= vocab_size <= 257024")
|
||||
raise ValueError(
|
||||
"When using LoRA, vocab size must be 32000 >= vocab_size <= 257024"
|
||||
)
|
||||
self.lora_a_stacked = torch.zeros(
|
||||
(
|
||||
max_loras,
|
||||
@@ -103,9 +110,10 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
max_loras,
|
||||
1,
|
||||
# Pad for kernel compatibility
|
||||
math.ceil(self.base_layer.vocab_size /
|
||||
lora_config.lora_vocab_padding_size) *
|
||||
lora_config.lora_vocab_padding_size,
|
||||
math.ceil(
|
||||
self.base_layer.vocab_size / lora_config.lora_vocab_padding_size
|
||||
)
|
||||
* lora_config.lora_vocab_padding_size,
|
||||
lora_config.max_lora_rank,
|
||||
),
|
||||
dtype=lora_config.lora_dtype,
|
||||
@@ -119,9 +127,8 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
)
|
||||
if self.sharded_to_full_mapping is not None:
|
||||
self.sharded_to_full_mapping_gpu = torch.tensor(
|
||||
self.sharded_to_full_mapping,
|
||||
device=self.device,
|
||||
dtype=torch.long)
|
||||
self.sharded_to_full_mapping, device=self.device, dtype=torch.long
|
||||
)
|
||||
else:
|
||||
self.sharded_to_full_mapping_gpu = None
|
||||
|
||||
@@ -139,17 +146,17 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
bias: Optional[torch.Tensor] = None,
|
||||
):
|
||||
self.reset_lora(index)
|
||||
self.lora_a_stacked[index,
|
||||
0, :lora_a.shape[0], :lora_a.shape[1]].copy_(
|
||||
lora_a, non_blocking=True)
|
||||
self.lora_b_stacked[index,
|
||||
0, :lora_b.shape[0], :lora_b.shape[1]].copy_(
|
||||
lora_b, non_blocking=True)
|
||||
self.lora_a_stacked[index, 0, : lora_a.shape[0], : lora_a.shape[1]].copy_(
|
||||
lora_a, non_blocking=True
|
||||
)
|
||||
self.lora_b_stacked[index, 0, : lora_b.shape[0], : lora_b.shape[1]].copy_(
|
||||
lora_b, non_blocking=True
|
||||
)
|
||||
if embeddings_tensor is not None:
|
||||
self.embeddings_tensors[
|
||||
index,
|
||||
:embeddings_tensor.shape[0],
|
||||
:embeddings_tensor.shape[1],
|
||||
: embeddings_tensor.shape[0],
|
||||
: embeddings_tensor.shape[1],
|
||||
] = embeddings_tensor
|
||||
|
||||
def _get_logits(
|
||||
@@ -195,41 +202,41 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
dtype=self.embeddings_tensors.dtype,
|
||||
device=self.embeddings_tensors.device,
|
||||
)
|
||||
torch.matmul(self.embeddings_tensors,
|
||||
hidden_states.T,
|
||||
out=lora_logits[:-1])
|
||||
torch.matmul(self.embeddings_tensors, hidden_states.T, out=lora_logits[:-1])
|
||||
|
||||
neg_inf, pos_inf = current_platform.get_infinity_values(
|
||||
lora_logits.dtype)
|
||||
neg_inf, pos_inf = current_platform.get_infinity_values(lora_logits.dtype)
|
||||
|
||||
lora_logits[-1] = neg_inf
|
||||
lora_logits = lora_logits.mT
|
||||
indices_padded = self.punica_wrapper.sampler_indices_padded
|
||||
|
||||
if current_platform.is_tpu() or current_platform.is_xpu():
|
||||
indices_padded = indices_padded[:logits.size(0)]
|
||||
indices_padded = indices_padded[: logits.size(0)]
|
||||
|
||||
lora_logits = (lora_logits.reshape(
|
||||
lora_logits.shape[0] * lora_logits.shape[1],
|
||||
lora_logits.shape[2],
|
||||
).index_select(0, indices_padded).nan_to_num_(nan=neg_inf,
|
||||
posinf=pos_inf,
|
||||
neginf=neg_inf))
|
||||
lora_logits = (
|
||||
lora_logits.reshape(
|
||||
lora_logits.shape[0] * lora_logits.shape[1],
|
||||
lora_logits.shape[2],
|
||||
)
|
||||
.index_select(0, indices_padded)
|
||||
.nan_to_num_(nan=neg_inf, posinf=pos_inf, neginf=neg_inf)
|
||||
)
|
||||
|
||||
logits[:,
|
||||
self.base_layer.org_vocab_size:self.base_layer.org_vocab_size +
|
||||
lora_logits.shape[1]] = lora_logits
|
||||
logits[
|
||||
:,
|
||||
self.base_layer.org_vocab_size : self.base_layer.org_vocab_size
|
||||
+ lora_logits.shape[1],
|
||||
] = lora_logits
|
||||
|
||||
lora_output: Optional[
|
||||
torch.Tensor] = self.punica_wrapper.add_lora_logits(
|
||||
logits, hidden_states, self.lora_a_stacked,
|
||||
self.lora_b_stacked, 1.0)
|
||||
lora_output: Optional[torch.Tensor] = self.punica_wrapper.add_lora_logits(
|
||||
logits, hidden_states, self.lora_a_stacked, self.lora_b_stacked, 1.0
|
||||
)
|
||||
|
||||
if not current_platform.can_update_inplace():
|
||||
logits = lora_output
|
||||
|
||||
# Remove paddings in vocab (if any).
|
||||
logits = logits[:, :self.base_layer.vocab_size]
|
||||
logits = logits[:, : self.base_layer.vocab_size]
|
||||
return logits
|
||||
|
||||
def forward(self, *args, **kwargs):
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
from .base import BaseLayerWithLoRA
|
||||
|
||||
|
||||
#TODO: Implement this
|
||||
# TODO: Implement this
|
||||
class QKVCrossParallelLinearWithLoRA(BaseLayerWithLoRA):
|
||||
pass
|
||||
|
||||
@@ -14,9 +14,10 @@ from .base_linear import BaseLinearLayerWithLoRA
|
||||
|
||||
|
||||
class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
|
||||
def __init__(self, base_layer: ReplicatedLinear) -> None:
|
||||
super().__init__(base_layer, )
|
||||
super().__init__(
|
||||
base_layer,
|
||||
)
|
||||
# To ensure interface compatibility, set to 1 always.
|
||||
self.output_size = self.base_layer.output_size
|
||||
self.n_slices = 1
|
||||
@@ -33,14 +34,12 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
- output
|
||||
- bias
|
||||
"""
|
||||
bias = (self.base_layer.bias
|
||||
if not self.base_layer.skip_bias_add else None)
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
|
||||
# Matrix multiply.
|
||||
output = self.apply(input_, bias)
|
||||
|
||||
output_bias = (self.base_layer.bias
|
||||
if self.base_layer.skip_bias_add else None)
|
||||
output_bias = self.base_layer.bias if self.base_layer.skip_bias_add else None
|
||||
|
||||
if not self.base_layer.return_bias:
|
||||
return output
|
||||
|
||||
@@ -8,8 +8,11 @@ import torch.nn as nn
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.distributed import (split_tensor_along_last_dim,
|
||||
tensor_model_parallel_all_reduce)
|
||||
from vllm.distributed import (
|
||||
split_tensor_along_last_dim,
|
||||
tensor_model_parallel_all_reduce,
|
||||
)
|
||||
|
||||
# yapf: disable
|
||||
from vllm.model_executor.layers.linear import RowParallelLinear
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
@@ -45,8 +45,7 @@ def _not_fully_sharded_can_replace(can_replace):
|
||||
|
||||
def dec(*args, **kwargs):
|
||||
decorate = kwargs.pop("decorate") if "decorate" in kwargs else True
|
||||
condition = (not kwargs["lora_config"].fully_sharded_loras
|
||||
if decorate else True)
|
||||
condition = not kwargs["lora_config"].fully_sharded_loras if decorate else True
|
||||
return can_replace(*args, **kwargs) and condition
|
||||
|
||||
return dec
|
||||
@@ -59,7 +58,8 @@ def _fully_sharded_can_replace(can_replace):
|
||||
"""
|
||||
|
||||
def dec(*args, **kwargs):
|
||||
return (can_replace(*args, **kwargs)
|
||||
and kwargs["lora_config"].fully_sharded_loras)
|
||||
return (
|
||||
can_replace(*args, **kwargs) and kwargs["lora_config"].fully_sharded_loras
|
||||
)
|
||||
|
||||
return dec
|
||||
|
||||
@@ -9,15 +9,13 @@ import torch.nn.functional as F
|
||||
from transformers import PretrainedConfig
|
||||
|
||||
from vllm.config.lora import LoRAConfig
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import (
|
||||
VocabParallelEmbedding)
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import VocabParallelEmbedding
|
||||
from vllm.platforms import current_platform
|
||||
|
||||
from .base import BaseLayerWithLoRA
|
||||
|
||||
|
||||
class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
|
||||
def __init__(self, base_layer: VocabParallelEmbedding) -> None:
|
||||
super().__init__()
|
||||
self.base_layer = base_layer
|
||||
@@ -25,24 +23,26 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
self.embeddings_weights: Optional[torch.Tensor]
|
||||
|
||||
def create_lora_weights(
|
||||
self,
|
||||
max_loras: int,
|
||||
lora_config: LoRAConfig,
|
||||
model_config: Optional[PretrainedConfig] = None) -> None:
|
||||
|
||||
self,
|
||||
max_loras: int,
|
||||
lora_config: LoRAConfig,
|
||||
model_config: Optional[PretrainedConfig] = None,
|
||||
) -> None:
|
||||
if self.base_layer.num_added_embeddings_per_partition > 0:
|
||||
# We can start adding lora weights
|
||||
self.embeddings_weights = self.base_layer.weight.data[
|
||||
self.base_layer.num_org_embeddings_per_partition:self.
|
||||
base_layer.num_org_embeddings_per_partition +
|
||||
self.base_layer.num_added_embeddings_per_partition]
|
||||
self.base_layer.num_org_embeddings_per_partition : self.base_layer.num_org_embeddings_per_partition
|
||||
+ self.base_layer.num_added_embeddings_per_partition
|
||||
]
|
||||
self.embeddings_slice = (
|
||||
self.base_layer.shard_indices.added_vocab_start_index -
|
||||
self.base_layer.org_vocab_size,
|
||||
self.base_layer.shard_indices.added_vocab_end_index -
|
||||
self.base_layer.org_vocab_size)
|
||||
self.base_layer.shard_indices.added_vocab_start_index
|
||||
- self.base_layer.org_vocab_size,
|
||||
self.base_layer.shard_indices.added_vocab_end_index
|
||||
- self.base_layer.org_vocab_size,
|
||||
)
|
||||
self.base_layer.weight.data[
|
||||
self.base_layer.num_org_embeddings_per_partition:].fill_(0)
|
||||
self.base_layer.num_org_embeddings_per_partition :
|
||||
].fill_(0)
|
||||
else:
|
||||
self.embeddings_slice = None
|
||||
self.embeddings_weights = None
|
||||
@@ -59,8 +59,7 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
self.lora_a_stacked = torch.zeros(
|
||||
(
|
||||
max_loras,
|
||||
self.base_layer.org_vocab_size +
|
||||
lora_config.lora_extra_vocab_size,
|
||||
self.base_layer.org_vocab_size + lora_config.lora_extra_vocab_size,
|
||||
lora_config.max_lora_rank,
|
||||
),
|
||||
dtype=lora_config.lora_dtype,
|
||||
@@ -97,31 +96,30 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
self.reset_lora(index)
|
||||
# NOTE self.lora_a_stacked is row-major, and lora_a is col-major,
|
||||
# so we need transpose here
|
||||
self.lora_a_stacked[index, :lora_a.shape[1], :lora_a.shape[0]].copy_(
|
||||
lora_a.T, non_blocking=True)
|
||||
self.lora_b_stacked[index,
|
||||
0, :lora_b.shape[0], :lora_b.shape[1]].copy_(
|
||||
lora_b, non_blocking=True)
|
||||
self.lora_a_stacked[index, : lora_a.shape[1], : lora_a.shape[0]].copy_(
|
||||
lora_a.T, non_blocking=True
|
||||
)
|
||||
self.lora_b_stacked[index, 0, : lora_b.shape[0], : lora_b.shape[1]].copy_(
|
||||
lora_b, non_blocking=True
|
||||
)
|
||||
if embeddings_tensor is not None:
|
||||
self.embeddings_tensors[
|
||||
index,
|
||||
:embeddings_tensor.shape[0],
|
||||
:embeddings_tensor.shape[1],
|
||||
: embeddings_tensor.shape[0],
|
||||
: embeddings_tensor.shape[1],
|
||||
].copy_(embeddings_tensor, non_blocking=True)
|
||||
if self.embeddings_slice is not None:
|
||||
# TODO(yard1): Optimize this copy, we don't need to copy
|
||||
# everything, just the modified part
|
||||
embeddings = self.embeddings_tensors.view(
|
||||
self.embeddings_tensors.shape[0] *
|
||||
self.embeddings_tensors.shape[1],
|
||||
self.embeddings_tensors.shape[0] * self.embeddings_tensors.shape[1],
|
||||
self.embeddings_tensors.shape[2],
|
||||
)[self.embeddings_slice[0]:self.embeddings_slice[1]]
|
||||
)[self.embeddings_slice[0] : self.embeddings_slice[1]]
|
||||
assert self.embeddings_weights is not None
|
||||
self.embeddings_weights[:embeddings.shape[0]].copy_(embeddings)
|
||||
self.embeddings_weights[: embeddings.shape[0]].copy_(embeddings)
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
added_tokens_mask = torch.where(x > self.base_layer.org_vocab_size - 1,
|
||||
1, 0)
|
||||
added_tokens_mask = torch.where(x > self.base_layer.org_vocab_size - 1, 1, 0)
|
||||
|
||||
# NB: Don't use torch.narrow here. torch.narrow triggers some
|
||||
# Dynamic Shape specialization in torch.compile
|
||||
@@ -133,26 +131,22 @@ class VocabParallelEmbeddingWithLoRA(BaseLayerWithLoRA):
|
||||
x + indices_1,
|
||||
self.lora_a_stacked_2d,
|
||||
)
|
||||
full_output = self.base_layer.forward(x +
|
||||
(indices_0 * added_tokens_mask))
|
||||
full_output = self.base_layer.forward(x + (indices_0 * added_tokens_mask))
|
||||
|
||||
full_output_org = full_output
|
||||
if full_output.ndim == 3:
|
||||
full_output = full_output.view(
|
||||
full_output.shape[0] * full_output.shape[1], -1)
|
||||
full_output.shape[0] * full_output.shape[1], -1
|
||||
)
|
||||
if full_lora_a_embeddings.ndim == 3:
|
||||
full_lora_a_embeddings = full_lora_a_embeddings.view(
|
||||
full_lora_a_embeddings.shape[0] *
|
||||
full_lora_a_embeddings.shape[1],
|
||||
full_lora_a_embeddings.shape[0] * full_lora_a_embeddings.shape[1],
|
||||
-1,
|
||||
)
|
||||
|
||||
lora_output: Optional[
|
||||
torch.Tensor] = self.punica_wrapper.add_lora_embedding(
|
||||
full_output,
|
||||
full_lora_a_embeddings,
|
||||
self.lora_b_stacked,
|
||||
add_input=True)
|
||||
lora_output: Optional[torch.Tensor] = self.punica_wrapper.add_lora_embedding(
|
||||
full_output, full_lora_a_embeddings, self.lora_b_stacked, add_input=True
|
||||
)
|
||||
|
||||
if not current_platform.can_update_inplace():
|
||||
full_output = lora_output
|
||||
|
||||
Reference in New Issue
Block a user