[Misc] Reduce LoRA-related static variable (#13166)
This commit is contained in:
@@ -27,6 +27,11 @@ from vllm.config import VllmConfig
|
||||
from vllm.distributed import get_tensor_model_parallel_world_size
|
||||
from vllm.distributed.utils import divide
|
||||
from vllm.logger import init_logger
|
||||
from vllm.lora.fully_sharded_layers import (
|
||||
ColumnParallelLinearWithShardedLoRA, RowParallelLinearWithShardedLoRA)
|
||||
from vllm.lora.layers import (ColumnParallelLinearWithLoRA,
|
||||
ReplicatedLinearWithLoRA,
|
||||
RowParallelLinearWithLoRA)
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
ReplicatedLinear,
|
||||
RowParallelLinear)
|
||||
@@ -103,6 +108,23 @@ def replace_linear_class(
|
||||
"rowwise": RowParallelLinear,
|
||||
}.get(style, ReplicatedLinear)
|
||||
|
||||
lora_linear_cls = {
|
||||
ColumnParallelLinear: {
|
||||
True: ColumnParallelLinearWithShardedLoRA, # fully sharded
|
||||
False: ColumnParallelLinearWithLoRA # not fully sharded
|
||||
},
|
||||
RowParallelLinear: {
|
||||
True: RowParallelLinearWithShardedLoRA,
|
||||
False: RowParallelLinearWithLoRA
|
||||
},
|
||||
# ReplicatedLinear doesn't support fully sharded LoRA yet,
|
||||
# so we use the same class for both cases.
|
||||
ReplicatedLinear: {
|
||||
True: ReplicatedLinearWithLoRA,
|
||||
False: ReplicatedLinearWithLoRA
|
||||
}
|
||||
}
|
||||
|
||||
class HFCompatibleLinear(vllm_linear_cls):
|
||||
"""
|
||||
Wrapper class that removes `output_bias` from returned output.
|
||||
@@ -111,6 +133,19 @@ def replace_linear_class(
|
||||
def forward(self, input: torch.Tensor) -> torch.Tensor:
|
||||
return super().forward(input)[0]
|
||||
|
||||
@classmethod
|
||||
def get_lora_class(cls, fully_sharded: bool = False):
|
||||
"""
|
||||
Get the LoRA class corresponding to the current transformer
|
||||
linear class.
|
||||
|
||||
Args:
|
||||
fully_sharded (bool): If True, select the LoRA class variant
|
||||
that supports fully sharded LoRA. Defaults to False.
|
||||
|
||||
"""
|
||||
return lora_linear_cls[vllm_linear_cls][fully_sharded]
|
||||
|
||||
return HFCompatibleLinear(
|
||||
input_size=linear.in_features,
|
||||
output_size=linear.out_features,
|
||||
|
||||
Reference in New Issue
Block a user