Add pydocstyle to the ruff rules (#52136)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com> Signed-off-by: Harry Mellor <hej.mellor@gmail.com> Co-authored-by: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 5
parent
804e5377aa
commit
8c1557a79c
@@ -244,8 +244,7 @@ class BaseLinearLayerWithLoRA(BaseLayerWithLoRA):
|
||||
def _apply_async_impl(
|
||||
self, x: torch.Tensor, bias: torch.Tensor | None = None
|
||||
) -> torch.Tensor:
|
||||
"""
|
||||
Forward pass with base linear and LoRA on separate CUDA streams
|
||||
"""Forward pass with base linear and LoRA on separate CUDA streams
|
||||
for overlap, using maybe_execute_in_parallel.
|
||||
Base layer runs on default stream; LoRA runs on aux stream.
|
||||
"""
|
||||
|
||||
@@ -83,8 +83,7 @@ def _mcp_apply(x, bias, layer: "ColumnParallelLinearWithLoRA"):
|
||||
|
||||
|
||||
class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
"""
|
||||
LoRA on top of ColumnParallelLinear layer.
|
||||
"""LoRA on top of ColumnParallelLinear layer.
|
||||
LoRA B is sliced for tensor parallelism.
|
||||
There are two types for the `base_layer`:
|
||||
1. ColumnParallelLinear, e.g.`dense_h_to_4h` in `FalconForCausalLM`.
|
||||
@@ -140,6 +139,7 @@ class ColumnParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
Returns:
|
||||
- output
|
||||
- bias
|
||||
|
||||
"""
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
|
||||
@@ -210,8 +210,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
The main reason for overriding this function is to enhance code
|
||||
"""The main reason for overriding this function is to enhance code
|
||||
maintainability.
|
||||
"""
|
||||
self.lora_config = lora_config
|
||||
@@ -268,8 +267,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
lora_a: list[torch.Tensor | None],
|
||||
lora_b: list[torch.Tensor | None],
|
||||
) -> tuple[list[torch.Tensor | None], list[torch.Tensor | None]]:
|
||||
"""
|
||||
Expand packed adapter groups when they don't match n_slices.
|
||||
"""Expand packed adapter groups when they don't match n_slices.
|
||||
E.g. in_proj_qkv (covers Q+K+V) + in_proj_z.
|
||||
|
||||
A None group member means that member was not adapted; the slice(s)
|
||||
@@ -388,8 +386,7 @@ class MergedColumnParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class QKVParallelLinearWithLoRA(ColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
ColumnParallelLinear layer that is specifically designed for
|
||||
"""ColumnParallelLinear layer that is specifically designed for
|
||||
qkv_proj. Certain models, such as chatglm3 and baichuan-7b,
|
||||
only contains a single LoRA within their qkv_proj layer.
|
||||
|
||||
@@ -493,8 +490,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
lora_config: LoRAConfig,
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
The main reason for overloading this function is to handle inconsistent
|
||||
"""The main reason for overloading this function is to handle inconsistent
|
||||
weight dimensions in qkv lora.
|
||||
"""
|
||||
super().create_lora_weights(max_loras, lora_config, model_config)
|
||||
@@ -521,8 +517,7 @@ class MergedQKVParallelLinearWithLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
|
||||
"""Differs from ColumnParallelLinearWithLoRA by slicing LoRA A also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
"""
|
||||
@@ -562,8 +557,7 @@ class ColumnParallelLinearWithShardedLoRA(ColumnParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from MergedColumnParallelLinearWithLoRA by slicing the
|
||||
"""Differs from MergedColumnParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -605,8 +599,7 @@ class MergedColumnParallelLinearWithShardedLoRA(MergedColumnParallelLinearWithLo
|
||||
|
||||
|
||||
class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from QKVParallelLinearWithLoRA by slicing the
|
||||
"""Differs from QKVParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -641,8 +634,7 @@ class QKVParallelLinearWithShardedLoRA(QKVParallelLinearWithLoRA):
|
||||
|
||||
|
||||
class MergedQKVParallelLinearWithShardedLoRA(MergedQKVParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from MergedQKVParallelLinearWithLoRA by slicing the
|
||||
"""Differs from MergedQKVParallelLinearWithLoRA by slicing the
|
||||
LoRA A's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the rank dim.
|
||||
@@ -741,7 +733,8 @@ class MergedColumnParallelLinearVariableSliceWithLoRA(
|
||||
lora_b: torch.Tensor | list[torch.Tensor],
|
||||
):
|
||||
"""Override to handle single tensor weights
|
||||
that need to be split into slices."""
|
||||
that need to be split into slices.
|
||||
"""
|
||||
self.reset_lora(index)
|
||||
|
||||
# Handle case where checkpoint has single tensor weights
|
||||
|
||||
@@ -248,7 +248,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""Initializes lora matrices."""
|
||||
|
||||
self._verify_ep_fs(lora_config)
|
||||
self.max_loras = lora_config.max_loras
|
||||
self.fully_sharded = lora_config.fully_sharded_loras
|
||||
@@ -294,9 +293,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
)
|
||||
|
||||
def _slice_w13_a(self, w13_lora_a: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
|
||||
"""
|
||||
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
|
||||
if self.tp_size == 1 or not self.fully_sharded:
|
||||
return w13_lora_a
|
||||
|
||||
@@ -321,9 +318,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
return w13_lora_b[:, start_idx:end_idx, :]
|
||||
|
||||
def _slice_w2_a(self, w2_lora_a: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
|
||||
"""
|
||||
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
|
||||
if self.tp_size == 1:
|
||||
return w2_lora_a
|
||||
# w2_lora_a shape (num_experts,rank,input_size)
|
||||
@@ -334,9 +329,7 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
return w2_lora_a[:, :, start_idx:end_idx]
|
||||
|
||||
def _slice_w2_b(self, w2_lora_b: torch.Tensor) -> torch.Tensor:
|
||||
"""
|
||||
Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA
|
||||
"""
|
||||
"""Applies to FusedMoEWithLoRA and FusedMoE3DWithLoRA."""
|
||||
if self.tp_size == 1 or not self.fully_sharded:
|
||||
return w2_lora_b
|
||||
# Based on S-LoRA, we slice W2 B along the hidden_size dim.
|
||||
@@ -475,7 +468,6 @@ class FusedMoEWithLoRA(BaseLayerWithLoRA):
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> bool:
|
||||
"""Returns True if the layer can be replaced by this LoRA layer."""
|
||||
|
||||
# source_layer is MoERunner
|
||||
moe_cls = maybe_get_oot_by_class(MoERunner)
|
||||
return isinstance(source_layer, moe_cls) and len(packed_modules_list) == 2
|
||||
@@ -522,7 +514,6 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
|
||||
model_config: PretrainedConfig | None = None,
|
||||
) -> None:
|
||||
"""Initializes lora matrices."""
|
||||
|
||||
if model_config is None:
|
||||
raise ValueError("model_config must be provided for MoE LoRA.")
|
||||
architectures = model_config.architectures
|
||||
@@ -609,30 +600,22 @@ class FusedMoE3DWithLoRA(FusedMoEWithLoRA):
|
||||
|
||||
@property
|
||||
def w13_input_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.w13_lora_a_stacked[0].shape[-1]
|
||||
|
||||
@property
|
||||
def w13_output_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.w13_lora_b_stacked[0].shape[-2] * self.tp_size
|
||||
|
||||
@property
|
||||
def w2_input_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.w2_lora_a_stacked[0].shape[-1] * self.tp_size
|
||||
|
||||
@property
|
||||
def w2_output_size(self):
|
||||
"""
|
||||
Full size
|
||||
"""
|
||||
"""Full size."""
|
||||
return self.hidden_size
|
||||
|
||||
@classmethod
|
||||
|
||||
@@ -18,8 +18,7 @@ from .base import BaseLayerWithLoRA
|
||||
|
||||
|
||||
class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
"""
|
||||
LoRA wrapper for LogitsProcessor, with extra logic to handle the
|
||||
"""LoRA wrapper for LogitsProcessor, with extra logic to handle the
|
||||
application of the LoRA adapter.
|
||||
|
||||
Args:
|
||||
@@ -30,6 +29,7 @@ class LogitsProcessorWithLoRA(BaseLayerWithLoRA):
|
||||
sharded_to_full_mapping: index mapping from sharded vocab to full vocab
|
||||
received from base_layer.get_sharded_to_full_mapping(). If None,
|
||||
no reindexing will be done.
|
||||
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
|
||||
@@ -25,7 +25,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
def forward(
|
||||
self, input_: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Forward of ReplicatedLinearWithLoRA
|
||||
"""Forward of ReplicatedLinearWithLoRA.
|
||||
|
||||
Args:
|
||||
input_: Tensor whose last dimension is `input_size`.
|
||||
@@ -33,6 +33,7 @@ class ReplicatedLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
Returns:
|
||||
- output
|
||||
- bias
|
||||
|
||||
"""
|
||||
bias = self.base_layer.bias if not self.base_layer.skip_bias_add else None
|
||||
|
||||
|
||||
@@ -42,7 +42,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
def forward(
|
||||
self, input_: torch.Tensor
|
||||
) -> torch.Tensor | tuple[torch.Tensor, torch.Tensor | None]:
|
||||
"""Forward of RowParallelLinear
|
||||
"""Forward of RowParallelLinear.
|
||||
|
||||
Args:
|
||||
input_: tensor whose last dimension is `input_size`. If
|
||||
@@ -52,6 +52,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
Returns:
|
||||
- output
|
||||
- bias
|
||||
|
||||
"""
|
||||
# set up backprop all-reduce.
|
||||
if self.base_layer.input_is_parallel:
|
||||
@@ -99,8 +100,7 @@ class RowParallelLinearWithLoRA(BaseLinearLayerWithLoRA):
|
||||
|
||||
|
||||
class RowParallelLinearWithShardedLoRA(RowParallelLinearWithLoRA):
|
||||
"""
|
||||
Differs from RowParallelLinearWithLoRA by slicing the
|
||||
"""Differs from RowParallelLinearWithLoRA by slicing the
|
||||
LoRA B's also.
|
||||
|
||||
Based on S-LoRA, slicing happens along the output dim.
|
||||
|
||||
@@ -74,8 +74,7 @@ def _get_lora_device(base_layer: nn.Module) -> torch.device:
|
||||
|
||||
|
||||
def _not_fully_sharded_can_replace(can_replace):
|
||||
"""
|
||||
decorator which adds the condition of not using fully sharded loras
|
||||
"""Decorator which adds the condition of not using fully sharded loras
|
||||
intended to wrap can_replace_layer()
|
||||
"""
|
||||
|
||||
@@ -88,8 +87,7 @@ def _not_fully_sharded_can_replace(can_replace):
|
||||
|
||||
|
||||
def _fully_sharded_can_replace(can_replace):
|
||||
"""
|
||||
decorator which adds the condition of fully sharded loras
|
||||
"""Decorator which adds the condition of fully sharded loras
|
||||
intended to wrap can_replace_layer()
|
||||
"""
|
||||
|
||||
|
||||
+20
-15
@@ -39,9 +39,7 @@ _EXPERTS_SEPARATOR = ".experts."
|
||||
|
||||
|
||||
def _is_remote_expert_key(raw_name: str, spec: "MoEEPLoadSpec") -> bool:
|
||||
"""
|
||||
Decide whether a checkpoint key belongs to a non-local expert.
|
||||
"""
|
||||
"""Decide whether a checkpoint key belongs to a non-local expert."""
|
||||
pos = raw_name.find(_EXPERTS_SEPARATOR)
|
||||
if pos < 0:
|
||||
return False
|
||||
@@ -69,15 +67,14 @@ class LoRAModel:
|
||||
*,
|
||||
modules_to_save: dict[str, LoRAFullModuleWeights] | None = None,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
lora_model_id: The integer id for the lora model.
|
||||
rank: lora rank.
|
||||
loras: module name -> weights for lora-replaced layers.
|
||||
is_3d_lora_weight: Whether the on-disk MoE adapter is in the 3D
|
||||
fused (gate_up_proj / down_proj) layout. Propagated from the
|
||||
originating LoRARequest. Only consulted by the LoRA model
|
||||
manager when enable_mixed_moe_lora_format is on.
|
||||
"""Args:
|
||||
lora_model_id: The integer id for the lora model.
|
||||
rank: lora rank.
|
||||
loras: module name -> weights for lora-replaced layers.
|
||||
is_3d_lora_weight: Whether the on-disk MoE adapter is in the 3D
|
||||
fused (gate_up_proj / down_proj) layout. Propagated from the
|
||||
originating LoRARequest. Only consulted by the LoRA model
|
||||
manager when enable_mixed_moe_lora_format is on.
|
||||
|
||||
"""
|
||||
self.id = lora_model_id
|
||||
@@ -93,7 +90,8 @@ class LoRAModel:
|
||||
def clone(self, lora_model_id: int) -> "LoRAModel":
|
||||
"""Return a copy of the object with different ids.
|
||||
|
||||
Will share the underlying tensors."""
|
||||
Will share the underlying tensors.
|
||||
"""
|
||||
return self.__class__(
|
||||
lora_model_id,
|
||||
rank=self.rank,
|
||||
@@ -103,7 +101,7 @@ class LoRAModel:
|
||||
)
|
||||
|
||||
def get_lora(self, module_name: str) -> LoRALayerWeights | None:
|
||||
"""Get LoRA for a given module by name"""
|
||||
"""Get LoRA for a given module by name."""
|
||||
return self.loras.get(module_name, None)
|
||||
|
||||
def get_module_to_save(self, module_name: str) -> LoRAFullModuleWeights | None:
|
||||
@@ -114,7 +112,7 @@ class LoRAModel:
|
||||
|
||||
@staticmethod
|
||||
def _should_skip_module(module_name: str, skip_prefixes: list[str]) -> bool:
|
||||
"""Check if a module should be skipped based on skip prefixes"""
|
||||
"""Check if a module should be skipped based on skip prefixes."""
|
||||
for prefix in skip_prefixes:
|
||||
if f".{prefix}" in module_name or module_name.startswith(prefix):
|
||||
return True
|
||||
@@ -222,6 +220,12 @@ class LoRAModel:
|
||||
a global counter.
|
||||
device: Device where the lora model is loaded.
|
||||
dtype: dtype of the lora model weights.
|
||||
model_vocab_size: Vocab size of the base model, used to size the
|
||||
embedding deltas.
|
||||
weights_mapper: Optional mapper rewriting checkpoint weight names
|
||||
to vLLM names.
|
||||
tensorizer_config_dict: Optional tensorizer config used to load
|
||||
the checkpoint via tensorizer instead of from disk.
|
||||
skip_prefixes: List of module name prefixes to skip during loading.
|
||||
Models can define this to skip modules not used in inference
|
||||
(e.g., MTP layers). Format: ["mtp."]
|
||||
@@ -233,6 +237,7 @@ class LoRAModel:
|
||||
|
||||
Returns:
|
||||
Loaded LoRA Model.
|
||||
|
||||
"""
|
||||
lora_tensor_path = os.path.join(lora_dir, "adapter_model.safetensors")
|
||||
lora_bin_file_path = os.path.join(lora_dir, "adapter_model.bin")
|
||||
|
||||
@@ -163,7 +163,6 @@ class PackedLoRALayerWeights(LoRALayerWeights):
|
||||
|
||||
If LoRA is None, it signifies that the submodule does not have a LoRA.
|
||||
"""
|
||||
|
||||
first_lora = next(lora for lora in loras if lora is not None)
|
||||
assert first_lora is not None
|
||||
rank = first_lora.rank
|
||||
|
||||
@@ -97,6 +97,9 @@ class LoRAModelManager:
|
||||
in a single batch.
|
||||
vocab_size: the vocab size of the model.
|
||||
lora_config: the LoRA configuration.
|
||||
device: the device the LoRA tensors are placed on.
|
||||
vllm_config: the vLLM config for this engine.
|
||||
|
||||
"""
|
||||
self.model: SupportsLoRAModel = model
|
||||
self.supported_lora_modules = get_supported_lora_modules(self.model)
|
||||
@@ -785,6 +788,7 @@ class LoRAModelManager:
|
||||
|
||||
Returns:
|
||||
True if the module passes the filter, False otherwise.
|
||||
|
||||
"""
|
||||
return is_in_target_modules(
|
||||
module_name,
|
||||
@@ -793,9 +797,7 @@ class LoRAModelManager:
|
||||
)
|
||||
|
||||
def _get_punica_wrapper(self, module_name: str) -> PunicaWrapperBase | None:
|
||||
"""
|
||||
Determine whether this module supports LoRA and which wrapper to use.
|
||||
"""
|
||||
"""Determine whether this module supports LoRA and which wrapper to use."""
|
||||
# For language model (early return)
|
||||
if not self.supports_mm:
|
||||
return self.punica_wrapper_mapping[DEFAULT_LANGUAGE_WRAPPER_KEY]
|
||||
@@ -1190,9 +1192,7 @@ class LoRAModelManager:
|
||||
return new_module_names[start:end]
|
||||
|
||||
def _build_moe_ep_load_spec(self) -> MoEEPLoadSpec | None:
|
||||
"""
|
||||
Per-rank slicing metadata for 2D RoutedEXperts LoRA modules.
|
||||
"""
|
||||
"""Per-rank slicing metadata for 2D RoutedEXperts LoRA modules."""
|
||||
if not self._use_ep or not self._is_moe:
|
||||
return None
|
||||
module = next(
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utilities for Punica kernel construction.
|
||||
"""
|
||||
"""Utilities for Punica kernel construction."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
@@ -21,8 +19,7 @@ def _accumulate_mm(
|
||||
group_n: tl.constexpr,
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Core matrix multiplication and accumulation logic with quantization support.
|
||||
"""Core matrix multiplication and accumulation logic with quantization support.
|
||||
|
||||
Args:
|
||||
tiled_a (tl.tensor): Loaded tile from A matrix
|
||||
@@ -36,8 +33,8 @@ def _accumulate_mm(
|
||||
group_k: Block size for K dimension in block-wise quantization
|
||||
group_n: Block size for N dimension in block-wise quantization
|
||||
use_fp8_w8a8: Whether using FP8 W8A8 quantization
|
||||
"""
|
||||
|
||||
"""
|
||||
if use_fp8_w8a8:
|
||||
if group_k > 0 and group_n > 0:
|
||||
# Block-wise quantization: scales are loaded per block
|
||||
@@ -85,8 +82,7 @@ def fp8_mm_k(
|
||||
USE_GDC: tl.constexpr,
|
||||
base_k,
|
||||
):
|
||||
"""
|
||||
FP8-compatible matrix multiplication kernel with quantization support.
|
||||
"""FP8-compatible matrix multiplication kernel with quantization support.
|
||||
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
B (k x n), iterate through the K dimension to compute the partial/complete
|
||||
matrix block product with proper dequantization.
|
||||
@@ -120,6 +116,7 @@ def fp8_mm_k(
|
||||
b_dtype: datatype of the B matrix
|
||||
USE_GDC: Whether to use PDL. True indicates use.
|
||||
base_k (int): Base offset along K dimension for current SPLIT_K group
|
||||
|
||||
"""
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
@@ -238,13 +235,11 @@ def do_shrink_kernel_fp8(
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice, compute the
|
||||
matrix product and store in the appropriate output location.
|
||||
"""
|
||||
|
||||
# Identify the lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
cur_lora_ptr = lora_ptr
|
||||
@@ -432,8 +427,7 @@ def do_expand_kernel_fp8(
|
||||
use_fp8_w8a8: tl.constexpr,
|
||||
per_channel_quant: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
FP8-compatible expand kernel for LoRA.
|
||||
"""FP8-compatible expand kernel for LoRA.
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice,
|
||||
@@ -449,7 +443,6 @@ def do_expand_kernel_fp8(
|
||||
- Per-channel quantization
|
||||
- Tensor-wise quantization
|
||||
"""
|
||||
|
||||
# ls_d*_ptr can be either an integer or a pointer
|
||||
if SAME_STRIDE:
|
||||
cur_lora_d0_stride = ls_d0_ptr
|
||||
|
||||
@@ -24,7 +24,7 @@ def _get_lora_id(
|
||||
top_k_num,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns lora_id"""
|
||||
"""Returns lora_id."""
|
||||
if naive_block_assignment:
|
||||
token_idx = pid_m // top_k_num
|
||||
return tl.load(token_lora_mapping_ptr + token_idx)
|
||||
@@ -41,7 +41,7 @@ def _get_expert_id(
|
||||
max_loras,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns expert_id"""
|
||||
"""Returns expert_id."""
|
||||
if naive_block_assignment:
|
||||
return tl.load(expert_ids_ptr + pid_m)
|
||||
else:
|
||||
@@ -61,7 +61,7 @@ def _get_token_offs(
|
||||
naive_block_assignment: tl.constexpr,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
):
|
||||
"""Returns token offsets"""
|
||||
"""Returns token offsets."""
|
||||
if naive_block_assignment:
|
||||
return tl.where(offs == 0, pid_m, num_valid_tokens)
|
||||
else:
|
||||
@@ -76,8 +76,7 @@ _LORA_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
|
||||
|
||||
|
||||
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
@@ -101,9 +100,7 @@ def _adjust_kernel_inputs(
|
||||
sorted_token_ids: torch.Tensor | None,
|
||||
expert_ids: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
helper function to adjust kernel inputs when sorted_token_ids is None
|
||||
"""
|
||||
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
|
||||
if sorted_token_ids is None:
|
||||
stride_tl = 0
|
||||
stride_el = 0
|
||||
|
||||
@@ -25,7 +25,7 @@ def _get_lora_id(
|
||||
top_k_num,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns lora_id"""
|
||||
"""Returns lora_id."""
|
||||
if naive_block_assignment:
|
||||
token_idx = pid_m // top_k_num
|
||||
return tl.load(token_lora_mapping_ptr + token_idx)
|
||||
@@ -42,7 +42,7 @@ def _get_expert_id(
|
||||
max_loras,
|
||||
naive_block_assignment: tl.constexpr,
|
||||
):
|
||||
"""Returns expert_id"""
|
||||
"""Returns expert_id."""
|
||||
if naive_block_assignment:
|
||||
return tl.load(expert_ids_ptr + pid_m)
|
||||
else:
|
||||
@@ -62,7 +62,7 @@ def _get_token_offs(
|
||||
naive_block_assignment: tl.constexpr,
|
||||
BLOCK_SIZE_M: tl.constexpr,
|
||||
):
|
||||
"""Returns token offsets"""
|
||||
"""Returns token offsets."""
|
||||
if naive_block_assignment:
|
||||
return tl.where(offs == 0, pid_m, num_valid_tokens)
|
||||
else:
|
||||
@@ -852,8 +852,7 @@ def _run_fused_moe_lora_small_batch(
|
||||
|
||||
|
||||
def _get_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
@@ -877,9 +876,7 @@ def _adjust_kernel_inputs(
|
||||
sorted_token_ids: torch.Tensor | None,
|
||||
expert_ids: torch.Tensor,
|
||||
):
|
||||
"""
|
||||
helper function to adjust kernel inputs when sorted_token_ids is None
|
||||
"""
|
||||
"""Helper function to adjust kernel inputs when sorted_token_ids is None."""
|
||||
if sorted_token_ids is None:
|
||||
stride_tl = 0
|
||||
stride_el = 0
|
||||
|
||||
@@ -1,19 +1,17 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Utilities for Punica kernel construction.
|
||||
"""
|
||||
"""Utilities for Punica kernel construction."""
|
||||
|
||||
from vllm.triton_utils import tl, triton
|
||||
|
||||
|
||||
@triton.jit
|
||||
def mm_k(
|
||||
a_ptr,
|
||||
b_ptr,
|
||||
ak_stride,
|
||||
bk_stride,
|
||||
offset_k,
|
||||
a_ptr: tl.tensor,
|
||||
b_ptr: tl.tensor,
|
||||
ak_stride: int,
|
||||
bk_stride: int,
|
||||
offset_k: tl.tensor,
|
||||
K: tl.constexpr,
|
||||
BLOCK_M: tl.constexpr,
|
||||
BLOCK_N: tl.constexpr,
|
||||
@@ -23,32 +21,34 @@ def mm_k(
|
||||
CAST_TYPE: tl.constexpr,
|
||||
b_dtype: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
base_k,
|
||||
base_k: int,
|
||||
):
|
||||
"""
|
||||
Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
"""Given a_ptr and b_ptr, that identify the rows of A (m x k) and columns of
|
||||
B (k x n), iterate, through the K dimension to compute the partial/complete
|
||||
matrix block product.
|
||||
If SPLIT_K == 1, the output m x n product is complete.
|
||||
If SPLIT_K > 1, the thread block computes partial outputs. The partial
|
||||
outputs are then atomically summed in the caller code.
|
||||
|
||||
Args:
|
||||
a_ptr: Array of pointers, identifying rows of A
|
||||
b_ptr: Array of pointers, identifying columns of B
|
||||
ak_stride: K dimension stride of the A matrix
|
||||
bk_stride: K dimension stride of the B matrix
|
||||
offset_k: Offsets within the current BLOCK_K tile along K
|
||||
K: Length of the K dimension
|
||||
BLOCK_M: M dimension of the output block m x n
|
||||
BLOCK_N: N dimension of the output block m x n
|
||||
BLOCK_K: K dimension atom
|
||||
EVEN_K: True if the blocks of A and B can be loaded without any
|
||||
masking.
|
||||
masking.
|
||||
SPLIT_K: Parameter signifying parallelism in the K dimension.
|
||||
CAST_TYPE: if True, cast the values from the A matrix to the B
|
||||
matrix dtype.
|
||||
matrix dtype.
|
||||
b_dtype: datatype of the B matrix
|
||||
USE_GDC: Whether to use PDL. True indicates use.
|
||||
base_k: Base offset along K dimension for current SPLIT_K group
|
||||
|
||||
"""
|
||||
accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32)
|
||||
|
||||
@@ -142,15 +142,13 @@ def do_expand_kernel(
|
||||
ADD_INPUTS: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice,
|
||||
compute the matrix product and store in the appropriate output location.
|
||||
Given that this is an expand kernel, we don't perform any split-K reduction
|
||||
as the K dimension is assumed to be small.
|
||||
"""
|
||||
|
||||
# ls_d*_ptr can be either an integer or a pointer
|
||||
if SAME_STRIDE:
|
||||
# integer
|
||||
@@ -268,13 +266,11 @@ def do_shrink_kernel(
|
||||
SLICE_NUM: tl.constexpr,
|
||||
USE_GDC: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
Given an array of integers that identifies the rows of A, ram,
|
||||
"""Given an array of integers that identifies the rows of A, ram,
|
||||
a lora index that identifies which LoRA to use from lora_ptr, lora_index,
|
||||
a slice_id that identifies the input/output slice, compute the
|
||||
matrix product and store in the appropriate output location.
|
||||
"""
|
||||
|
||||
# Identify the lora_ptr from slice_id.
|
||||
if SLICE_NUM == 1:
|
||||
# current lora ptr
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -21,8 +20,7 @@ _EXPAND_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], torch.tensor] = {}
|
||||
|
||||
|
||||
def _get_expand_lora_scale_ptr(lora_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
"""`_EXPAND_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
`profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
@@ -91,9 +89,7 @@ def _lora_expand_kernel_fp8(
|
||||
per_channel_quant: tl.constexpr,
|
||||
launch_pdl: tl.constexpr,
|
||||
):
|
||||
"""
|
||||
FP8-compatible expand kernel wrapper.
|
||||
"""
|
||||
"""FP8-compatible expand kernel wrapper."""
|
||||
cta_n_num = tl.cdiv(N, BLOCK_N)
|
||||
cta_m_num = tl.cdiv(M, BLOCK_M)
|
||||
|
||||
@@ -192,8 +188,7 @@ def _lora_expand_fp8(
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
FP8-compatible LoRA expand operation.
|
||||
"""FP8-compatible LoRA expand operation.
|
||||
|
||||
Args:
|
||||
inputs: Input tensor from shrink operation [num_slices, num_tokens, lora_rank]
|
||||
@@ -206,6 +201,8 @@ def _lora_expand_fp8(
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
num_active_loras: Number of active LoRAs. Accepted for API parity with
|
||||
the non-FP8 kernel and unused here.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
@@ -216,6 +213,7 @@ def _lora_expand_fp8(
|
||||
group_n (int, optional): Block size for N in block-wise quantization.
|
||||
use_fp8_w8a8 (bool, optional): Whether to use FP8 W8A8 quantization.
|
||||
per_channel_quant (bool, optional): Whether to use per-channel quantization.
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -147,32 +146,31 @@ def _lora_expand(
|
||||
offset_start: int = 0,
|
||||
add_inputs: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): input tensor
|
||||
lora_b_weights (list[torch.Tensor]): lora'b weight
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
Defaults to 0.
|
||||
add_inputs (bool, optional): Whether to add the input tensor to the
|
||||
output tensor. Defaults to False.
|
||||
"""
|
||||
"""Args:
|
||||
inputs (torch.Tensor): input tensor
|
||||
lora_b_weights (list[torch.Tensor]): lora'b weight
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
offset_start (int, optional): Offset start for output_tensor.
|
||||
Defaults to 0.
|
||||
add_inputs (bool, optional): Whether to add the input tensor to the
|
||||
output tensor. Defaults to False.
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
# None of the inputs require LoRA.
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
LoRA kernels metadata preparation utilities.
|
||||
"""
|
||||
"""LoRA kernels metadata preparation utilities."""
|
||||
|
||||
import bisect
|
||||
from dataclasses import dataclass, field
|
||||
@@ -107,14 +105,13 @@ class LoRAKernelMeta:
|
||||
self.num_active_loras_cpu.fill_(0)
|
||||
|
||||
def prepare_tensors(self, token_lora_mapping: torch.Tensor) -> None:
|
||||
"""
|
||||
Prepare kernel metadata tensors for the current forward pass.
|
||||
"""Prepare kernel metadata tensors for the current forward pass.
|
||||
|
||||
Args:
|
||||
token_lora_mapping (torch.Tensor): Tensor containing lora indices
|
||||
for each input token.
|
||||
"""
|
||||
|
||||
"""
|
||||
self._reset()
|
||||
|
||||
# Check and record no-lora case.
|
||||
@@ -180,8 +177,7 @@ class LoRAKernelMeta:
|
||||
torch.Tensor,
|
||||
torch.Tensor,
|
||||
]:
|
||||
"""
|
||||
This function returns the kernel metadata required for the current
|
||||
"""This function returns the kernel metadata required for the current
|
||||
forward pass execution of the kernel. The function returns all the
|
||||
metadata required by the kernel, in order, as a tuple, so it can be
|
||||
unpacked directly during the lora_shrink/lora_expand function call.
|
||||
@@ -189,6 +185,10 @@ class LoRAKernelMeta:
|
||||
Args:
|
||||
token_nums (int): Number of input tokens in the current forward
|
||||
pass of the kernel.
|
||||
specialize_active_lora (bool): If True, report the actual active
|
||||
LoRA count; otherwise report the padded default, which keeps
|
||||
the kernel launch shape stable.
|
||||
|
||||
"""
|
||||
if specialize_active_lora:
|
||||
num_active_loras = self.num_active_loras_cpu
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -20,8 +19,7 @@ _SHRINK_LORA_SCALE_PTR_DICT: dict[tuple[int, ...], tuple] = {}
|
||||
def _get_shrink_lora_scale_ptr(
|
||||
lora_scale_weights: list[torch.Tensor], device: torch.device
|
||||
):
|
||||
"""
|
||||
`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
"""`_SHRINK_LORA_SCALE_PTR_DICT` collects the required information during
|
||||
`profile_run`. After this, it remains constant and subsequent usage is
|
||||
through LUT.
|
||||
|
||||
@@ -241,23 +239,23 @@ def _lora_shrink_fp8(
|
||||
use_fp8_w8a8: bool = False,
|
||||
per_channel_quant: bool = False,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
|
||||
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
|
||||
output_tensor: Output tensor (FP16/BF16/FP32)
|
||||
token_lora_mapping: Token to LoRA ID mapping
|
||||
token_indices_sorted_by_lora_ids: Sorted token indices
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
scaling: LoRA scaling factor
|
||||
a_scale: Activation quantization scales
|
||||
b_scale: Weight quantization scales per slice
|
||||
group_k: Block size for K dimension quantization
|
||||
group_n: Block size for N dimension quantization
|
||||
use_fp8_w8a8: Whether to use FP8 weights and activations
|
||||
per_channel_quant: Whether to use per-channel quantization
|
||||
"""Args:
|
||||
inputs: FP8 or FP16/BF16 input tensor [num_tokens, hidden_size]
|
||||
lora_a_weights: List of FP8 or FP16/BF16 LoRA A weights per slice
|
||||
output_tensor: Output tensor (FP16/BF16/FP32)
|
||||
token_lora_mapping: Token to LoRA ID mapping
|
||||
token_indices_sorted_by_lora_ids: Sorted token indices
|
||||
num_tokens_per_lora: Number of tokens per LoRA
|
||||
lora_token_start_loc: Start location for each LoRA's tokens
|
||||
lora_ids: LoRA IDs to process
|
||||
scaling: LoRA scaling factor
|
||||
a_scale: Activation quantization scales
|
||||
b_scale: Weight quantization scales per slice
|
||||
group_k: Block size for K dimension quantization
|
||||
group_n: Block size for N dimension quantization
|
||||
use_fp8_w8a8: Whether to use FP8 weights and activations
|
||||
per_channel_quant: Whether to use per-channel quantization
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -142,32 +141,31 @@ def _lora_shrink(
|
||||
num_active_loras: torch.Tensor, # CPU tensor [1], number of active LoRAs
|
||||
scaling: float,
|
||||
) -> None:
|
||||
"""
|
||||
Args:
|
||||
inputs (torch.Tensor): Input tensor
|
||||
lora_a_weights (list[torch.Tensor]): LoRA weights
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
|
||||
number of active LoRAs. Stored as a tensor (not int) so
|
||||
torch.compile treats it as dynamic rather than a constant.
|
||||
scaling (float): Scaling factor.
|
||||
"""
|
||||
"""Args:
|
||||
inputs (torch.Tensor): Input tensor
|
||||
lora_a_weights (list[torch.Tensor]): LoRA weights
|
||||
output_tensor (torch.Tensor): output tensor
|
||||
token_lora_mapping (torch.Tensor): A tensor mapping each input token
|
||||
to the lora-id related to that token. A value of -1 indicates that
|
||||
LoRA doesn't apply to that token.
|
||||
token_indices_sorted_by_lora_ids (torch.Tensor): Row/Token indices from
|
||||
the A matrix grouped by LoRA IDs.
|
||||
num_tokens_per_lora (torch.Tensor): num_tokens_per_lora[i] is the number
|
||||
of tokens that are to be processed by LoRA ID lora_ids[i]
|
||||
lora_token_start_loc (torch.Tensor): A cumulative sum of
|
||||
num_tokens_per_lora. lora_token_start_loc[0] is always 0 so that
|
||||
lora_token_start_loc[i], along with num_tokens_per_lora[i]
|
||||
identifies the region in token_indices_sorted_by_lora_ids that
|
||||
LoRA lora_ids[i] should process.
|
||||
lora_ids (torch.Tensor): LoRA ids to process.
|
||||
no_lora_flag_cpu (torch.Tensor): A CPU tensor of size 1, that indicates
|
||||
if there are any requests that require LoRA.
|
||||
num_active_loras (torch.Tensor): A CPU tensor of size 1, containing the
|
||||
number of active LoRAs. Stored as a tensor (not int) so
|
||||
torch.compile treats it as dynamic rather than a constant.
|
||||
scaling (float): Scaling factor.
|
||||
|
||||
"""
|
||||
assert no_lora_flag_cpu.numel() == 1
|
||||
if no_lora_flag_cpu.item():
|
||||
# None of the inputs require LoRA.
|
||||
|
||||
@@ -23,8 +23,7 @@ _LORA_B_PTR_DICT: dict[tuple[int, ...], tuple[torch.tensor, ...]] = {}
|
||||
|
||||
|
||||
def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
|
||||
"""
|
||||
`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_A_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
@@ -75,14 +74,12 @@ def _get_lora_a_ptr(lora_a_weights: list[torch.Tensor], device: torch.device):
|
||||
def _get_lora_b_ptr(
|
||||
lora_weights: list[torch.Tensor], offset_start: int, device: torch.device
|
||||
):
|
||||
"""
|
||||
`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
|
||||
"""`_LORA_B_PTR_DICT` collects the required information during `profile_run`,
|
||||
After this, it remains constant and subsequent usage is through LUT.
|
||||
Refer to:
|
||||
https://github.com/triton-lang/triton/blob/release/3.1.x/python/tutorials/08-grouped-gemm.py
|
||||
|
||||
"""
|
||||
|
||||
key = tuple(lora_weight.data_ptr() for lora_weight in lora_weights)
|
||||
if values := _LORA_B_PTR_DICT.get(key):
|
||||
return values
|
||||
@@ -317,9 +314,7 @@ def get_lora_op_configs(
|
||||
|
||||
@lru_cache
|
||||
def supports_pdl(device: torch.device | None = None) -> bool:
|
||||
"""
|
||||
Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py
|
||||
"""
|
||||
"""Refer to: https://github.com/triton-lang/triton/blob/v3.5.0/python/tutorials/11-programmatic-dependent-launch.py."""
|
||||
# PDL requires compute capability SM90 or above
|
||||
|
||||
return (
|
||||
|
||||
@@ -18,8 +18,7 @@ logger = init_logger(__name__)
|
||||
|
||||
@dataclass
|
||||
class PEFTHelper:
|
||||
"""
|
||||
A helper class for PEFT configurations, specifically designed for LoRA.
|
||||
"""A helper class for PEFT configurations, specifically designed for LoRA.
|
||||
This class handles configuration validation, compatibility checks for
|
||||
various LoRA implementations.
|
||||
"""
|
||||
@@ -40,9 +39,7 @@ class PEFTHelper:
|
||||
vllm_max_position_embeddings: int | None = field(default=False)
|
||||
|
||||
def _validate_features(self) -> list[str]:
|
||||
"""
|
||||
Check if there are any unsupported LoRA features.
|
||||
"""
|
||||
"""Check if there are any unsupported LoRA features."""
|
||||
error_msg = []
|
||||
if self.modules_to_save:
|
||||
unsupported_modules = [
|
||||
@@ -124,8 +121,7 @@ class PEFTHelper:
|
||||
return cls.from_dict(config)
|
||||
|
||||
def validate_legal(self, lora_config: LoRAConfig) -> None:
|
||||
"""
|
||||
Validates the LoRA configuration settings against application
|
||||
"""Validates the LoRA configuration settings against application
|
||||
constraints and requirements.
|
||||
"""
|
||||
error_msg = self._validate_features()
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
@@ -20,9 +19,7 @@ if TYPE_CHECKING:
|
||||
|
||||
|
||||
class PunicaWrapperABC(ABC):
|
||||
"""
|
||||
PunicaWrapper ABC.
|
||||
"""
|
||||
"""PunicaWrapper ABC."""
|
||||
|
||||
@abstractmethod
|
||||
def update_metadata(
|
||||
@@ -33,9 +30,7 @@ class PunicaWrapperABC(ABC):
|
||||
vocab_size: int,
|
||||
**kwargs,
|
||||
) -> None:
|
||||
"""
|
||||
Update the lora-related metadata
|
||||
"""
|
||||
"""Update the lora-related metadata"""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -47,10 +42,7 @@ class PunicaWrapperABC(ABC):
|
||||
scale: float,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_a.
|
||||
"""
|
||||
|
||||
"""Performs GEMM for multiple slices of lora_a."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -64,9 +56,7 @@ class PunicaWrapperABC(ABC):
|
||||
add_inputs=True,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_b.
|
||||
"""
|
||||
"""Performs GEMM for multiple slices of lora_b."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -78,8 +68,7 @@ class PunicaWrapperABC(ABC):
|
||||
add_inputs: bool = True,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Applies lora specifically for VocabParallelEmbeddingWithLoRA,
|
||||
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA,
|
||||
and this layer only requires the expand operation.
|
||||
"""
|
||||
raise NotImplementedError
|
||||
@@ -97,10 +86,7 @@ class PunicaWrapperABC(ABC):
|
||||
buffer: tuple[torch.Tensor, ...] | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Applicable to linear-related lora.
|
||||
"""
|
||||
|
||||
"""Applicable to linear-related lora."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -115,9 +101,7 @@ class PunicaWrapperABC(ABC):
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
"""
|
||||
"""Applies lora specifically for LogitsProcessorWithLoRA."""
|
||||
raise NotImplementedError
|
||||
|
||||
@abstractmethod
|
||||
@@ -134,8 +118,7 @@ class PunicaWrapperABC(ABC):
|
||||
|
||||
|
||||
class PunicaWrapperBase(PunicaWrapperABC):
|
||||
"""
|
||||
PunicaWrapperBase is designed to manage and provide metadata for the punica
|
||||
"""PunicaWrapperBase is designed to manage and provide metadata for the punica
|
||||
kernel. The main function is to maintain the state information for
|
||||
Multi-LoRA, and to provide the interface for the punica.
|
||||
"""
|
||||
@@ -249,8 +232,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
def prefill_metadata(
|
||||
self,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int]:
|
||||
"""
|
||||
This property provides a convenient way to access the necessary
|
||||
"""This property provides a convenient way to access the necessary
|
||||
metadata for prefill-related kernel computations.
|
||||
1. seq_start_locs: Tensor of sequence start positions.
|
||||
2. seq_lengths: Tensor of sequence lengths.
|
||||
@@ -271,8 +253,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
|
||||
@property
|
||||
def token_lora_indices(self) -> torch.Tensor:
|
||||
"""
|
||||
This property provides the lora indices corresponding to each token
|
||||
"""This property provides the lora indices corresponding to each token
|
||||
in the batch. An index of -1 means no lora should be applied.
|
||||
"""
|
||||
token_lora_len = self.indices_len[0]
|
||||
@@ -280,8 +261,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
|
||||
@property
|
||||
def sampler_indices(self) -> torch.Tensor:
|
||||
"""
|
||||
This property is used to access the lora indices specifically for
|
||||
"""This property is used to access the lora indices specifically for
|
||||
LogitsProcessorWithLoRA.
|
||||
"""
|
||||
sampler_indices_len = self.indices_len[1]
|
||||
@@ -289,9 +269,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
|
||||
@property
|
||||
def sampler_indices_padded(self) -> torch.Tensor:
|
||||
"""
|
||||
This property provides access to padded sampler indices.
|
||||
"""
|
||||
"""This property provides access to padded sampler indices."""
|
||||
indices_padded_len = self.indices_len[2]
|
||||
return self._sampler_indices_padded[:indices_padded_len]
|
||||
|
||||
@@ -321,8 +299,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
scale: float,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_a.
|
||||
"""Performs GEMM for multiple slices of lora_a.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -349,8 +326,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
add_inputs=True,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_b.
|
||||
"""Performs GEMM for multiple slices of lora_b.
|
||||
|
||||
Semantics:
|
||||
offset = offset_start
|
||||
@@ -380,8 +356,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
add_inputs: bool = True,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
and this layer only requires the expand operation.
|
||||
Semantics:
|
||||
y += x @ lora_b_stacked
|
||||
@@ -391,6 +366,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
x (torch.Tensor): Input tensor.
|
||||
lora_b_stacked (torch.Tensor): lora_b's weights.
|
||||
add_inputs (bool): Default to True.
|
||||
|
||||
"""
|
||||
# TODO: implement it based on torch ops
|
||||
raise NotImplementedError
|
||||
@@ -408,8 +384,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
buffer: tuple[torch.Tensor, ...] | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Applicable to linear-related lora.
|
||||
"""Applicable to linear-related lora.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -428,6 +403,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
scale (float): Scaling factor.
|
||||
output_slices (tuple[int, ...]): Every slice's size.
|
||||
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
|
||||
|
||||
"""
|
||||
# TODO: implement it based on torch ops
|
||||
raise NotImplementedError
|
||||
@@ -444,8 +420,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
) -> torch.Tensor | None:
|
||||
"""
|
||||
Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
"""Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
|
||||
Semantics:
|
||||
buffer = (x @ lora_a_stacked) * scale
|
||||
@@ -458,6 +433,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
lora_b_stacked (torch.Tensor):lora_b's weights.
|
||||
scale (float): Scaling factor.
|
||||
buffer (Optional[torch.Tensor]):Default to None.
|
||||
|
||||
"""
|
||||
# TODO: implement it based on torch ops
|
||||
raise NotImplementedError
|
||||
@@ -474,8 +450,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
pad_sorted_ids: bool = False,
|
||||
naive_block_assignment: bool = False,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
"""Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
mixture-of-experts (MoE) execution.
|
||||
"""
|
||||
# TODO: implement it based on torch ops
|
||||
@@ -501,8 +476,7 @@ class PunicaWrapperBase(PunicaWrapperABC):
|
||||
offset: int = 0,
|
||||
token_lora_mapping: torch.Tensor | None = None,
|
||||
):
|
||||
"""
|
||||
Performs a fused forward computation for LoRA of
|
||||
"""Performs a fused forward computation for LoRA of
|
||||
Mixture-of-Experts (MoE) layer.
|
||||
"""
|
||||
# TODO: implement it based on torch ops
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
|
||||
from collections.abc import Callable
|
||||
from typing import Any
|
||||
|
||||
import torch
|
||||
|
||||
@@ -20,8 +21,7 @@ from .punica_base import PunicaWrapperBase
|
||||
# The platforms that are compatible with the PyTorch-native implementation can
|
||||
# inherit this class
|
||||
class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
"""
|
||||
PunicaWrapperCPU is designed to manage and provide metadata for the punica
|
||||
"""PunicaWrapperCPU is designed to manage and provide metadata for the punica
|
||||
kernel. The main function is to maintain the state information for
|
||||
Multi-LoRA, and to provide the interface for the pytorch punica ops.
|
||||
"""
|
||||
@@ -133,12 +133,10 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
y_slice_size: int,
|
||||
add_inputs: bool = True,
|
||||
):
|
||||
"""
|
||||
Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
|
||||
"""Perform the ` y[:,y_offset:y_offset+y_slice_size]+=x@w_t_all`
|
||||
computation, which is suitable for the
|
||||
GEMM of lora'b.
|
||||
"""
|
||||
|
||||
expand_slice_fun: Callable = (
|
||||
self._expand_slice_prefill if self.is_prefill else self._expand_slice_decode
|
||||
)
|
||||
@@ -147,8 +145,7 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
def _apply_shrink(
|
||||
self, y: torch.Tensor, x: torch.Tensor, w_t_all: torch.Tensor, scale: float
|
||||
):
|
||||
"""
|
||||
Perform the ` y+=x@w_t_all` computation, which is suitable for the
|
||||
"""Perform the ` y+=x@w_t_all` computation, which is suitable for the
|
||||
GEMM of lora'a.
|
||||
When `is_prefill is` true, it indicates that it is currently the
|
||||
prefill stage, and the `_shrink_prefill` function should be called.
|
||||
@@ -169,10 +166,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
x: torch.Tensor,
|
||||
lora_a_stacked: tuple[torch.Tensor, ...],
|
||||
scale: float,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_a.
|
||||
"""Performs GEMM for multiple slices of lora_a.
|
||||
When `is_prefill is` true, it indicates that it is currently the
|
||||
prefill stage, and the `_shrink_prefill` function should be called.
|
||||
Otherwise, it is the decode stage, and the _shrink_decode function
|
||||
@@ -187,8 +183,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensor
|
||||
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
|
||||
scale (float): Scaling factor for the operation
|
||||
"""
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
x = x.view(-1, x.shape[-1])
|
||||
# TODO fuse these kernels
|
||||
for slice_idx in range(len(lora_a_stacked)):
|
||||
@@ -202,10 +199,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
output_slices: tuple[int, ...],
|
||||
offset_start: int = 0,
|
||||
add_inputs=True,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_b.
|
||||
"""Performs GEMM for multiple slices of lora_b.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_b_stacked)):
|
||||
@@ -218,7 +214,10 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
x (Union[tuple[torch.Tensor, ...], torch.Tensor]): Input tensors
|
||||
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
|
||||
output_slices (tuple[int, ...]): Every slice's size
|
||||
offset_start (int): The starting position of y, defaults to 0.
|
||||
add_inputs (bool): Defaults to True.
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
@@ -241,10 +240,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
x: torch.Tensor,
|
||||
lora_b_stacked: torch.Tensor,
|
||||
add_inputs: bool = True,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
|
||||
Semantics:
|
||||
y += x @ lora_b_stacked
|
||||
@@ -254,8 +252,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensor.
|
||||
lora_b_stacked (torch.Tensor): lora_b's weights.
|
||||
add_inputs (bool): Default to True.
|
||||
"""
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
# Embedding layer only need expand op
|
||||
expand_fun: Callable = (
|
||||
self._expand_prefill if self.is_prefill else self._expand_decode
|
||||
@@ -272,10 +271,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
output_slices: tuple[int, ...],
|
||||
*,
|
||||
buffer: tuple[torch.Tensor, ...] | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applicable to linear-related lora.
|
||||
"""Applicable to linear-related lora.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -294,8 +292,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
scale (float): Scaling factor.
|
||||
output_slices (tuple[int, ...]): Every slice's size.
|
||||
buffer (Optional[tuple[torch.Tensor, ...]]): Defaults to None.
|
||||
"""
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
|
||||
|
||||
if buffer is None:
|
||||
@@ -320,10 +319,9 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
scale,
|
||||
*,
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
"""Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
|
||||
Semantics:
|
||||
buffer = (x @ lora_a_stacked) * scale
|
||||
@@ -336,6 +334,8 @@ class PunicaWrapperCPU(PunicaWrapperBase):
|
||||
lora_b_stacked (torch.Tensor):lora_b's weights.
|
||||
scale (float): Scaling factor.
|
||||
buffer (Optional[torch.Tensor]):Default to None.
|
||||
**kwargs: Forwarded to `add_shrink` and `add_expand`.
|
||||
|
||||
"""
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
"""
|
||||
|
||||
from typing import final
|
||||
from typing import Any, final
|
||||
|
||||
import torch
|
||||
|
||||
@@ -32,8 +31,7 @@ from .punica_base import PunicaWrapperBase
|
||||
|
||||
@final
|
||||
class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
"""
|
||||
PunicaWrapperGPU is designed to manage and provide metadata for the punica
|
||||
"""PunicaWrapperGPU is designed to manage and provide metadata for the punica
|
||||
kernel. The main function is to maintain the state information for
|
||||
Multi-LoRA, and to provide the interface for the punica triton kernel.
|
||||
"""
|
||||
@@ -96,10 +94,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
x: torch.Tensor,
|
||||
lora_a_stacked: tuple[torch.Tensor, ...],
|
||||
scale: float,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_a.
|
||||
"""Performs GEMM for multiple slices of lora_a.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -110,8 +107,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensor
|
||||
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
|
||||
scale (float): Scaling factor for the operation
|
||||
"""
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
x = x.view(-1, x.shape[-1])
|
||||
lora_shrink(
|
||||
x,
|
||||
@@ -131,10 +129,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
output_slices: tuple[int, ...],
|
||||
offset_start: int = 0,
|
||||
add_inputs=True,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_b.
|
||||
"""Performs GEMM for multiple slices of lora_b.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_b_stacked)):
|
||||
@@ -147,9 +144,12 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensors
|
||||
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
|
||||
output_slices (tuple[int, ...]): Every slice's size
|
||||
offset_start (int): The starting position of y, defaults to 0.
|
||||
add_inputs (bool): If True, add LoRA output to y; if False, write
|
||||
LoRA-only output to y (used for dual-stream when base and LoRA
|
||||
run on different CUDA streams). Defaults to True.
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
@@ -177,10 +177,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
x: torch.Tensor,
|
||||
lora_b_stacked: torch.Tensor,
|
||||
add_inputs: bool = True,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
|
||||
Semantics:
|
||||
y += x @ lora_b_stacked
|
||||
@@ -190,8 +189,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensor.
|
||||
lora_b_stacked (torch.Tensor): lora_b's weights.
|
||||
add_inputs (bool): Default to True.
|
||||
"""
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
lora_expand(
|
||||
x.unsqueeze(dim=0),
|
||||
(lora_b_stacked,),
|
||||
@@ -213,10 +213,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
output_slices: tuple[int, ...],
|
||||
*,
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applicable to linear-related lora.
|
||||
"""Applicable to linear-related lora.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -226,6 +225,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
@ lora_b_stacked[indices[i], layer_idx, :, :]
|
||||
* scale
|
||||
).squeeze(0)
|
||||
|
||||
Args:
|
||||
y (torch.Tensor): Output tensor. Will be changed in-place.
|
||||
x (torch.Tensor): Input tensor
|
||||
@@ -234,8 +234,10 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
scale (float): Scaling factor.
|
||||
output_slices (tuple[int, ...]): Every slice's size.
|
||||
buffer (Optional[torch.Tensor]): Defaults to None.
|
||||
"""
|
||||
**kwargs: Accepts `add_inputs` (bool, defaults to True); any other
|
||||
keys are ignored.
|
||||
|
||||
"""
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
|
||||
|
||||
assert buffer is None, (
|
||||
@@ -275,10 +277,9 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
scale,
|
||||
*,
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
"""Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
|
||||
Semantics:
|
||||
buffer = (x @ lora_a_stacked) * scale
|
||||
@@ -291,6 +292,8 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
lora_b_stacked (torch.Tensor): lora_b's weights.
|
||||
scale (float): Scaling factor.
|
||||
buffer (Optional[torch.Tensor]): Default to None.
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
@@ -367,8 +370,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
naive_block_assignment: bool = False,
|
||||
token_lora_mapping: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
"""Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
mixture-of-experts (MoE) execution.
|
||||
|
||||
When `token_lora_mapping` is provided, it overrides the global mapping
|
||||
@@ -467,9 +469,7 @@ class PunicaWrapperGPU(PunicaWrapperBase):
|
||||
token_lora_mapping: torch.Tensor | None = None,
|
||||
add_inputs: bool = True,
|
||||
):
|
||||
"""
|
||||
Performs a fused forward computation for LoRA of Mixture-of-Experts (MoE) layer.
|
||||
"""
|
||||
"""Perform a fused forward computation for a LoRA MoE layer."""
|
||||
(
|
||||
token_lora_mapping_meta,
|
||||
_,
|
||||
|
||||
@@ -1,13 +1,12 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
||||
"""
|
||||
Based on:
|
||||
"""Based on:
|
||||
Chen, L., Ye, Z., Wu, Y., Zhuo, D., Ceze, L., & Krishnamurthy, A. (2023).
|
||||
Punica: Multi-Tenant LoRA Serving.
|
||||
https://arxiv.org/abs/2310.18547
|
||||
"""
|
||||
|
||||
from typing import final
|
||||
from typing import Any, final
|
||||
|
||||
import torch
|
||||
|
||||
@@ -29,8 +28,7 @@ from .punica_base import PunicaWrapperBase
|
||||
|
||||
@final
|
||||
class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
"""
|
||||
PunicaWrapperXPU is designed to manage and provide metadata for the punica
|
||||
"""PunicaWrapperXPU is designed to manage and provide metadata for the punica
|
||||
kernel. The main function is to maintain the state information for
|
||||
Multi-LoRA, and to provide the interface for the punica ipex kernel.
|
||||
"""
|
||||
@@ -135,10 +133,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
x: torch.Tensor,
|
||||
lora_a_stacked: tuple[torch.Tensor, ...],
|
||||
scale: float,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
):
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_a.
|
||||
"""Performs GEMM for multiple slices of lora_a.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -149,8 +146,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensor
|
||||
lora_a_stacked (tuple[torch.Tensor, ...]): lora_a's weights
|
||||
scale (float): Scaling factor for the operation
|
||||
"""
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
x = x.view(-1, x.shape[-1])
|
||||
for slice_idx in range(len(lora_a_stacked)):
|
||||
self._apply_shrink(y[slice_idx], x, lora_a_stacked[slice_idx], scale)
|
||||
@@ -163,10 +161,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
output_slices: tuple[int, ...],
|
||||
offset_start: int = 0,
|
||||
add_inputs=True,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Performs GEMM for multiple slices of lora_b.
|
||||
"""Performs GEMM for multiple slices of lora_b.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_b_stacked)):
|
||||
@@ -179,7 +176,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensors
|
||||
lora_b_stacked (tuple[torch.Tensor, ...]): lora_b's weight
|
||||
output_slices (tuple[int, ...]): Every slice's size
|
||||
offset_start (int): The starting position of y, defaults to 0.
|
||||
add_inputs (bool): Defaults to True.
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
@@ -206,10 +206,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
x: torch.Tensor,
|
||||
lora_b_stacked: torch.Tensor,
|
||||
add_inputs: bool = True,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
"""Applies lora specifically for VocabParallelEmbeddingWithLoRA.
|
||||
|
||||
Semantics:
|
||||
y += x @ lora_b_stacked
|
||||
@@ -219,6 +218,8 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
x (torch.Tensor): Input tensor.
|
||||
lora_b_stacked (torch.Tensor): lora_b's weights.
|
||||
add_inputs (bool): Default to True.
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
token_lora_indices = self._get_token_lora_indices(x)
|
||||
bgmv_expand(x, lora_b_stacked, y, token_lora_indices, add_inputs)
|
||||
@@ -233,10 +234,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
output_slices: tuple[int, ...],
|
||||
*,
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applicable to linear-related lora.
|
||||
"""Applicable to linear-related lora.
|
||||
|
||||
Semantics:
|
||||
for i in range(len(lora_a_stacked)):
|
||||
@@ -246,6 +246,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
@ lora_b_stacked[indices[i], layer_idx, :, :]
|
||||
* scale
|
||||
).squeeze(0)
|
||||
|
||||
Args:
|
||||
y (torch.Tensor): Output tensor. Will be changed in-place.
|
||||
x (torch.Tensor): Input tensor
|
||||
@@ -254,8 +255,10 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
scale (float): Scaling factor.
|
||||
output_slices (tuple[int, ...]): Every slice's size.
|
||||
buffer (Optional[torch.Tensor]): Defaults to None.
|
||||
"""
|
||||
**kwargs: Accepts `add_inputs` (bool, defaults to True); any other
|
||||
keys are ignored.
|
||||
|
||||
"""
|
||||
assert len(lora_a_stacked) == len(lora_b_stacked) == len(output_slices)
|
||||
|
||||
assert buffer is None, (
|
||||
@@ -287,9 +290,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
|
||||
@property
|
||||
def sampler_indices_padded(self) -> torch.Tensor:
|
||||
"""
|
||||
This property provides access to padded sampler indices.
|
||||
"""
|
||||
"""This property provides access to padded sampler indices."""
|
||||
return self._sampler_indices_padded[:]
|
||||
|
||||
def add_lora_logits(
|
||||
@@ -301,10 +302,9 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
scale,
|
||||
*,
|
||||
buffer: torch.Tensor | None = None,
|
||||
**kwargs,
|
||||
**kwargs: Any,
|
||||
) -> None:
|
||||
"""
|
||||
Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
"""Applies lora specifically for LogitsProcessorWithLoRA.
|
||||
|
||||
Semantics:
|
||||
buffer = (x @ lora_a_stacked) * scale
|
||||
@@ -317,6 +317,8 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
lora_b_stacked (torch.Tensor): lora_b's weights.
|
||||
scale (float): Scaling factor.
|
||||
buffer (Optional[torch.Tensor]): Default to None.
|
||||
**kwargs: Unused; accepted for compatibility with the base class signature.
|
||||
|
||||
"""
|
||||
y_org = y
|
||||
y = y.view(-1, y.shape[-1])
|
||||
@@ -366,8 +368,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
naive_block_assignment: bool = False,
|
||||
token_lora_mapping: torch.Tensor | None = None,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:
|
||||
"""
|
||||
Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
"""Aligns tokens and experts into block-sized chunks for LoRA-based
|
||||
mixture-of-experts (MoE) execution.
|
||||
|
||||
When `token_lora_mapping` is provided, it overrides the global mapping
|
||||
@@ -466,9 +467,7 @@ class PunicaWrapperXPU(PunicaWrapperBase):
|
||||
token_lora_mapping: torch.Tensor | None = None,
|
||||
add_inputs: bool = True,
|
||||
):
|
||||
"""
|
||||
Performs a fused forward computation for LoRA of Mixture-of-Experts (MoE) layer.
|
||||
"""
|
||||
"""Perform a fused forward computation for a LoRA MoE layer."""
|
||||
(
|
||||
token_lora_mapping_meta,
|
||||
_,
|
||||
|
||||
@@ -15,15 +15,13 @@ if TYPE_CHECKING:
|
||||
def compute_meta(
|
||||
token_lora_tensor: torch.Tensor,
|
||||
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, bool]:
|
||||
"""
|
||||
Get the information required for the sgmv kernel. With the features:
|
||||
"""Get the information required for the sgmv kernel. With the features:
|
||||
1. If consecutive requests in the batch use the same LoRA, this function
|
||||
will combine them into a single request, improving sgmv kernel inference
|
||||
performance.
|
||||
2. At the beginning of each prefill stage inference, recalculations are
|
||||
needed based on the input, but only once.
|
||||
"""
|
||||
|
||||
lora_indices_tensor, seq_length_tensor = torch.unique_consecutive(
|
||||
token_lora_tensor, return_counts=True
|
||||
)
|
||||
@@ -65,6 +63,7 @@ def convert_mapping(
|
||||
lora_index_to_id: List mapping LoRA ids to LoRA indices.
|
||||
max_loras: Maximum number of LoRAs.
|
||||
vocab_size: Model vocab size.
|
||||
device: Device the returned index tensors are created on.
|
||||
|
||||
Returns:
|
||||
A tuple of tensors:
|
||||
@@ -83,6 +82,7 @@ def convert_mapping(
|
||||
indices_len: List of lengths of the above tensors. It contains
|
||||
(base_indices, sampler_indices, sampler_indices_padded,
|
||||
embeddings_indices).
|
||||
|
||||
"""
|
||||
index_mapping_indices: list[int] = list(mapping.index_mapping).copy()
|
||||
embedding_indices = index_mapping_indices.copy()
|
||||
|
||||
@@ -10,8 +10,7 @@ class LoRARequest(
|
||||
omit_defaults=True, # type: ignore[call-arg]
|
||||
array_like=True,
|
||||
): # type: ignore[call-arg]
|
||||
"""
|
||||
Request for a LoRA adapter.
|
||||
"""Request for a LoRA adapter.
|
||||
|
||||
lora_int_id must be globally unique for a given adapter.
|
||||
This is currently not enforced in vLLM.
|
||||
@@ -56,16 +55,14 @@ class LoRARequest(
|
||||
return self.lora_path
|
||||
|
||||
def __eq__(self, value: object) -> bool:
|
||||
"""
|
||||
Overrides the equality method to compare LoRARequest
|
||||
"""Overrides the equality method to compare LoRARequest
|
||||
instances based on lora_name. This allows for identification
|
||||
and comparison lora adapter across engines.
|
||||
"""
|
||||
return isinstance(value, self.__class__) and self.lora_name == value.lora_name
|
||||
|
||||
def __hash__(self) -> int:
|
||||
"""
|
||||
Overrides the hash method to hash LoRARequest instances
|
||||
"""Overrides the hash method to hash LoRARequest instances
|
||||
based on lora_name. This ensures that LoRARequest instances
|
||||
can be used in hash-based collections such as sets and dictionaries,
|
||||
identified by their names across engines.
|
||||
|
||||
@@ -36,6 +36,7 @@ class LoRAResolver(ABC):
|
||||
Returns:
|
||||
Optional[LoRARequest]: The resolved LoRA model information, or None
|
||||
if the LoRA model cannot be found.
|
||||
|
||||
"""
|
||||
pass
|
||||
|
||||
@@ -54,9 +55,11 @@ class _LoRAResolverRegistry:
|
||||
resolver: LoRAResolver,
|
||||
) -> None:
|
||||
"""Register a LoRA resolver.
|
||||
|
||||
Args:
|
||||
resolver_name: Name to register the resolver under.
|
||||
resolver: The LoRA resolver instance to register.
|
||||
|
||||
"""
|
||||
if resolver_name in self.resolvers:
|
||||
logger.warning(
|
||||
@@ -70,12 +73,16 @@ class _LoRAResolverRegistry:
|
||||
|
||||
def get_resolver(self, resolver_name: str) -> LoRAResolver:
|
||||
"""Get a registered resolver instance by name.
|
||||
|
||||
Args:
|
||||
resolver_name: Name of the resolver to get.
|
||||
|
||||
Returns:
|
||||
The resolver instance.
|
||||
|
||||
Raises:
|
||||
KeyError: If the resolver is not found in the registry.
|
||||
|
||||
"""
|
||||
if resolver_name not in self.resolvers:
|
||||
raise KeyError(
|
||||
|
||||
+12
-15
@@ -49,8 +49,7 @@ logger = init_logger(__name__)
|
||||
|
||||
|
||||
def get_captured_lora_counts(max_loras: int, specialize: bool) -> list[int]:
|
||||
"""
|
||||
Returns num_active_loras values for cudagraph capture.
|
||||
"""Returns num_active_loras values for cudagraph capture.
|
||||
|
||||
When specialize=True: powers of 2 up to max_loras, plus max_loras + 1.
|
||||
When specialize=False: just [max_loras + 1].
|
||||
@@ -170,7 +169,7 @@ def parse_fine_tuned_lora_name(
|
||||
) -> tuple[str, bool]:
|
||||
"""Parse the name of lora weights.
|
||||
|
||||
args:
|
||||
Args:
|
||||
name: the name of the fine-tuned LoRA, e.g.
|
||||
base_model.model.dense1.weight
|
||||
weights_mapper: maps the name of weight, e.g.
|
||||
@@ -179,8 +178,8 @@ def parse_fine_tuned_lora_name(
|
||||
tuple(module_name, is_lora_a):
|
||||
module_name: the name of the module, e.g. model.dense1,
|
||||
is_lora_a whether the tensor is lora_a or lora_b.
|
||||
"""
|
||||
|
||||
"""
|
||||
# LoRA weight qualified name usually starts with `base_model.model.`,
|
||||
# so we remove the prefix `base_model.model.` to make the following
|
||||
# mapping correctly.
|
||||
@@ -231,10 +230,7 @@ def is_base_embedding_weights(name: str) -> bool:
|
||||
|
||||
|
||||
def get_supported_lora_modules(model: nn.Module) -> list[str]:
|
||||
"""
|
||||
In vLLM, all linear layers support LoRA.
|
||||
"""
|
||||
|
||||
"""In vLLM, all linear layers support LoRA."""
|
||||
supported_lora_modules: set[str] = set()
|
||||
for name, module in model.named_modules():
|
||||
# get the embedding modules if the module's embedding_modules
|
||||
@@ -268,6 +264,7 @@ def is_supported_lora_module(
|
||||
|
||||
Returns:
|
||||
True if the module is supported, False otherwise.
|
||||
|
||||
"""
|
||||
module_suffix = module_name.rsplit(".", 1)[-1]
|
||||
if module_suffix not in supported_lora_modules:
|
||||
@@ -305,6 +302,7 @@ def is_in_target_modules(
|
||||
|
||||
Returns:
|
||||
True if the module passes the filter, False otherwise.
|
||||
|
||||
"""
|
||||
if target_modules is None:
|
||||
return True
|
||||
@@ -329,22 +327,21 @@ def is_in_target_modules(
|
||||
|
||||
|
||||
def get_adapter_absolute_path(lora_path: str) -> str:
|
||||
"""
|
||||
Resolves the given lora_path to an absolute local path.
|
||||
"""Resolves the given lora_path to an absolute local path.
|
||||
|
||||
If the lora_path is identified as a Hugging Face model identifier,
|
||||
it will download the model and return the local snapshot path.
|
||||
Otherwise, it treats the lora_path as a local file path and
|
||||
converts it to an absolute path.
|
||||
|
||||
Parameters:
|
||||
lora_path (str): The path to the lora model, which can be an absolute path,
|
||||
a relative path, or a Hugging Face model identifier.
|
||||
Args:
|
||||
lora_path (str): The path to the lora model, which can be an absolute
|
||||
path, a relative path, or a Hugging Face model identifier.
|
||||
|
||||
Returns:
|
||||
str: The resolved absolute local path to the lora model.
|
||||
"""
|
||||
str: The resolved absolute local path to the lora model.
|
||||
|
||||
"""
|
||||
# Check if the path is an absolute path. Return it no matter exists or not.
|
||||
if os.path.isabs(lora_path):
|
||||
return lora_path
|
||||
|
||||
@@ -28,7 +28,8 @@ class WorkerLoRAManager:
|
||||
"""WorkerLoRAManager that manages LoRA models on the worker side.
|
||||
|
||||
Every request, the requested LoRAs will be loaded (unless they are already
|
||||
loaded), and every other LoRA will be unloaded."""
|
||||
loaded), and every other LoRA will be unloaded.
|
||||
"""
|
||||
|
||||
_manager_cls: type[LoRAModelManager] = LoRAModelManager
|
||||
|
||||
@@ -74,7 +75,8 @@ class WorkerLoRAManager:
|
||||
@contextmanager
|
||||
def dummy_lora_cache(self):
|
||||
"""Use this context manager to reuse the dummy lora model
|
||||
to avoid creating it repeatedly."""
|
||||
to avoid creating it repeatedly.
|
||||
"""
|
||||
self._cached_dummy_lora = None
|
||||
yield
|
||||
self._cached_dummy_lora = False
|
||||
@@ -249,7 +251,8 @@ class LRUCacheWorkerLoRAManager(WorkerLoRAManager):
|
||||
|
||||
Uses an LRU Cache. Every request, the requested LoRAs will be loaded
|
||||
(unless they are already loaded) and least recently used LoRAs will
|
||||
be unloaded if the cache is above capacity."""
|
||||
be unloaded if the cache is above capacity.
|
||||
"""
|
||||
|
||||
_manager_cls: type[LRUCacheLoRAModelManager] = LRUCacheLoRAModelManager
|
||||
|
||||
|
||||
Reference in New Issue
Block a user